diff options
255 files changed, 7549 insertions, 4053 deletions
diff --git a/.gitignore b/.gitignore index e4cec0ef6..57b8a68d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ * !**/ !*.* + +# Cache nimcache/ +rnimcache/ +dnimcache/ *.o !/icons/*.o diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..76c94c8e7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,61 @@ +image: ubuntu:16.04 + +stages: + - pre-build + - build + - deploy + - test + +.linux_set_path: &linux_set_path_def + before_script: + - export PATH=$(pwd)/bin:$PATH + tags: + - linux + +.windows_set_path: &win_set_path_def + before_script: + - set PATH=%CD%\bin;%PATH% + tags: + - windows + + +build-windows: + stage: build + script: + - ci\build.bat + artifacts: + paths: + - bin\nim.exe + - bin\nimd.exe + - compiler\nim.exe + - koch.exe + expire_in: 1 week + tags: + - windows + +deploy-windows: + stage: deploy + script: + - koch.exe winrelease + artifacts: + paths: + - build/*.exe + - build/*.zip + expire_in: 1 week + tags: + - windows + - fast + + + +test-windows: + stage: test + <<: *win_set_path_def + script: + - call ci\deps.bat + - nim c --taintMode:on tests\testament\tester + - tests\testament\tester.exe --pedantic all + tags: + - windows + - fast + diff --git a/.travis.yml b/.travis.yml index 0ebeeb995..ebf287502 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ script: - nimble install zip - nimble install opengl - nimble install sdl1 - - nimble install jester + - nimble install jester@#head - nimble install niminst - nim c --taintMode:on tests/testament/tester - tests/testament/tester --pedantic all diff --git a/bootstrap.sh b/bootstrap.sh index dd551b52d..5d05ddbf8 100644 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -3,7 +3,7 @@ set -e set -x if [ ! -e csources/.git ]; then - git clone --depth 1 git://github.com/nim-lang/csources.git csources + git clone --depth 1 https://github.com/nim-lang/csources.git csources fi cd "csources" diff --git a/build_tools.sh b/build_tools.sh new file mode 100644 index 000000000..4d867b0ad --- /dev/null +++ b/build_tools.sh @@ -0,0 +1,3 @@ +./bin/nim c --noNimblePath -p:compiler -o:./bin/nimble dist/nimble/src/nimble.nim +./bin/nim c --noNimblePath -p:compiler -o:./bin/nimsuggest dist/nimsuggest/nimsuggest.nim +./bin/nim c -o:./bin/nimgrep tools/nimgrep.nim diff --git a/ci/build.bat b/ci/build.bat new file mode 100644 index 000000000..cdce8d3d2 --- /dev/null +++ b/ci/build.bat @@ -0,0 +1,14 @@ +REM Some debug info +echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%." +gcc -v + +git clone --depth 1 https://github.com/nim-lang/csources.git +cd csources +call build64.bat +cd .. +set PATH=%CD%\bin;%PATH% +nim -v +nim c koch +koch.exe boot +copy bin/nim bin/nimd +koch.exe boot -d:release diff --git a/ci/build.sh b/ci/build.sh new file mode 100644 index 000000000..a0fee1497 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,15 @@ +sh ci/deps.sh + +# Build from C sources. +git clone --depth 1 https://github.com/nim-lang/csources.git +cd csources +sh build.sh +cd .. +# Add Nim to the PATH +export PATH=$(pwd)/bin:$PATH +# Bootstrap. +nim -v +nim c koch +./koch boot +cp bin/nim bin/nimd +./koch boot -d:release diff --git a/ci/deps.bat b/ci/deps.bat new file mode 100644 index 000000000..bda1fe14f --- /dev/null +++ b/ci/deps.bat @@ -0,0 +1,4 @@ +nim e install_nimble.nims +nim e tests/test_nimscript.nims +nimble update +nimble install -y zip opengl sdl1 jester@#head niminst diff --git a/ci/deps.sh b/ci/deps.sh new file mode 100644 index 000000000..3385a213b --- /dev/null +++ b/ci/deps.sh @@ -0,0 +1,17 @@ +# Some debug info +echo "Running on $CI_RUNNER_ID ($CI_RUNNER_DESCRIPTION) with tags $CI_RUNNER_TAGS." + +# Packages +apt-get update -qq +apt-get install -y -qq build-essential git libcurl4-openssl-dev libsdl1.2-dev libgc-dev nodejs fasm + +gcc -v + +fasm -v +export PATH=$(pwd)/bin:$PATH + +# Nimble deps +nim e install_nimble.nims +nim e tests/test_nimscript.nims +nimble update +nimble install zip opengl sdl1 jester@#head niminst diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat new file mode 100644 index 000000000..657b2b90a --- /dev/null +++ b/ci/nsis_build.bat @@ -0,0 +1,62 @@ +REM - Run the full testsuite; tests\testament\tester all + +REM - Uncomment the list of changes in news.txt +REM - write a news ticker entry +REM - Update the version + +REM - Generate the full docs; koch web0 +REM - Generate the installers; +REM - Update the version in system.nim +REM - Test the installers +REM - Tag the release +REM - Merge devel into master +REM - Update csources + +set NIMVER=%1 + +Rem Build -docs file: +koch web0 +cd web\upload +7z a -tzip docs-%NIMVER%.zip *.html +move /y docs-%NIMVER%.zip download +cd ..\.. + +Rem Build .zip file: +rem koch csources -d:release +rem koch xz -d:release +rem move /y build\nim-%NIMVER%.zip web\upload\download + + +rem Grab C sources and nimsuggest +git clone --depth 1 https://github.com/nim-lang/csources.git + +set PATH=%CD%\bin;%PATH% + +ReM Build Win32 version: + +set PATH=C:\Users\araq\projects\mingw32\bin;%PATH% +cd csources +call build.bat +cd .. +ReM Rebuilding koch is necessary because it uses its pointer size to determine +ReM which mingw link to put in the NSIS installer. +nim c --out:koch_temp koch || exit /b +koch_temp boot -d:release || exit /b +koch_temp nsis -d:release || exit /b +koch_temp zip -d:release || exit /b +dir build +move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b +move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b + + +ReM Build Win64 version: +set PATH=C:\Users\araq\projects\mingw64\bin;%PATH% +cd csources +call build64.bat +cd .. +nim c --out:koch_temp koch || exit /b +koch_temp boot -d:release || exit /b +koch_temp nsis -d:release || exit /b +koch_temp zip -d:release || exit /b +move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b +move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b diff --git a/compiler.nimble b/compiler.nimble index c63fe49bf..66d2ee7c2 100644 --- a/compiler.nimble +++ b/compiler.nimble @@ -1,6 +1,6 @@ [Package] name = "compiler" -version = "0.14.3" +version = "0.15.1" author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" @@ -8,4 +8,4 @@ license = "MIT" InstallDirs = "doc, compiler" [Deps] -Requires: "nim >= 0.13.0" +Requires: "nim >= 0.14.0" diff --git a/compiler/aliases.nim b/compiler/aliases.nim index 4b592ee60..4186900ec 100644 --- a/compiler/aliases.nim +++ b/compiler/aliases.nim @@ -46,7 +46,8 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult = if compareTypes(a, b, dcEqIgnoreDistinct): return arYes case a.kind of tyObject: - result = isPartOfAux(a.sons[0], b, marker) + if a.sons[0] != nil: + result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker) if result == arNo: result = isPartOfAux(a.n, b, marker) of tyGenericInst, tyDistinct: result = isPartOfAux(lastSon(a), b, marker) diff --git a/compiler/ast.nim b/compiler/ast.nim index 277a21ba5..d8939fc60 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -437,7 +437,7 @@ type nfExplicitCall # x.y() was used instead of x.y nfExprCall # this is an attempt to call a regular expression nfIsRef # this node is a 'ref' node; used for the VM - nfIsCursor # this node is attached a cursor; used for idetools + nfPreventCg # this node should be ignored by the codegen TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 28) @@ -725,10 +725,7 @@ type s*: TStorageLoc flags*: TLocFlags # location's flags t*: PType # type of location - r*: Rope # rope value of location (code generators) - heapRoot*: Rope # keeps track of the enclosing heap object that - # owns this location (required by GC algorithms - # employing heap snapshots or sliding views) + r*: Rope # rope value of location (code generators) # ---------------- end of backend information ------------------------------ @@ -746,10 +743,6 @@ type TInstantiation* = object sym*: PSym concreteTypes*: seq[PType] - usedBy*: seq[int32] # list of modules using the generic - # needed in caas mode for purging the cache - # XXX: it's possible to switch to a - # simple ref count here compilesId*: CompilesId PInstantiation* = ref TInstantiation @@ -767,7 +760,6 @@ type case kind*: TSymKind of skType, skGenericParam: typeInstCache*: seq[PType] - typScope*: PScope of routineKinds: procInstCache*: seq[PInstantiation] gcUnsafetyReason*: PSym # for better error messages wrt gcsafe @@ -862,9 +854,6 @@ type key*, val*: RootRef TPairSeq* = seq[TPair] - TTable* = object # the same as table[PObject] of PObject - counter*: int - data*: TPairSeq TIdPair* = object key*: PIdObj @@ -936,7 +925,7 @@ const skMacro, skTemplate, skConverter, skEnumField, skLet, skStub, skAlias} PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, - nfIsRef, nfIsCursor, nfLL} + nfIsRef, nfPreventCg, nfLL} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 @@ -1108,12 +1097,6 @@ proc copyIdTable*(dest: var TIdTable, src: TIdTable) = newSeq(dest.data, len(src.data)) for i in countup(0, high(src.data)): dest.data[i] = src.data[i] -proc copyTable*(dest: var TTable, src: TTable) = - dest.counter = src.counter - if isNil(src.data): return - setLen(dest.data, len(src.data)) - for i in countup(0, high(src.data)): dest.data[i] = src.data[i] - proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet) = dest.counter = src.counter if isNil(src.data): return @@ -1327,10 +1310,6 @@ proc initStrTable*(x: var TStrTable) = proc newStrTable*: TStrTable = initStrTable(result) -proc initTable(x: var TTable) = - x.counter = 0 - newSeq(x.data, StartSize) - proc initIdTable*(x: var TIdTable) = x.counter = 0 newSeq(x.data, StartSize) @@ -1429,6 +1408,7 @@ proc copyNode*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1447,6 +1427,7 @@ proc shallowCopy*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1466,6 +1447,7 @@ proc copyTree*(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -1511,19 +1493,9 @@ proc hasSubnodeWith*(n: PNode, kind: TNodeKind): bool = return true result = false -proc replaceSons(n: PNode, oldKind, newKind: TNodeKind) = - for i in countup(0, sonsLen(n) - 1): - if n.sons[i].kind == oldKind: n.sons[i].kind = newKind - -proc sonsNotNil(n: PNode): bool = - for i in countup(0, sonsLen(n) - 1): - if n.sons[i] == nil: - return false - result = true - proc getInt*(a: PNode): BiggestInt = case a.kind - of nkIntLit..nkUInt64Lit: result = a.intVal + of nkCharLit..nkUInt64Lit: result = a.intVal else: internalError(a.info, "getInt") result = 0 diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 3ca44ea7e..7c07b2995 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -31,17 +31,6 @@ proc objectSetIncl*(t: var TObjectSet, obj: RootRef) proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool # more are not needed ... -# ----------------------- (key, val)-Hashtables ---------------------------- -proc tablePut*(t: var TTable, key, val: RootRef) -proc tableGet*(t: TTable, key: RootRef): RootRef -type - TCmpProc* = proc (key, closure: RootRef): bool {.nimcall.} # true if found - -proc tableSearch*(t: TTable, key, closure: RootRef, - comparator: TCmpProc): RootRef - # return val as soon as comparator returns true; if this never happens, - # nil is returned - # ----------------------- str table ----------------------------------------- proc strTableContains*(t: TStrTable, n: PSym): bool proc strTableAdd*(t: var TStrTable, n: PSym) @@ -251,20 +240,6 @@ proc symToYamlAux(n: PSym, marker: var IntSet, indent, maxRecDepth: int): Rope proc typeToYamlAux(n: PType, marker: var IntSet, indent, maxRecDepth: int): Rope -proc strTableToYaml(n: TStrTable, marker: var IntSet, indent: int, - maxRecDepth: int): Rope = - var istr = rspaces(indent + 2) - result = rope("[") - var mycount = 0 - for i in countup(0, high(n.data)): - if n.data[i] != nil: - if mycount > 0: add(result, ",") - addf(result, "$N$1$2", - [istr, symToYamlAux(n.data[i], marker, indent + 2, maxRecDepth - 1)]) - inc(mycount) - if mycount > 0: addf(result, "$N$1", [rspaces(indent)]) - add(result, "]") - assert(mycount == n.counter) proc ropeConstr(indent: int, c: openArray[Rope]): Rope = # array of (name, value) pairs @@ -463,9 +438,6 @@ proc debug(n: PType) = proc debug(n: PNode) = echo($debugTree(n, 0, 100)) -const - EmptySeq = @[] - proc nextTry(h, maxHash: Hash): Hash = result = ((5 * h) + 1) and maxHash # For any initial h in range(maxHash), repeating that maxHash times @@ -519,55 +491,6 @@ proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool = inc(t.counter) result = false -proc tableRawGet(t: TTable, key: RootRef): int = - var h: Hash = hashNode(key) and high(t.data) # start with real hash value - while t.data[h].key != nil: - if t.data[h].key == key: - return h - h = nextTry(h, high(t.data)) - result = -1 - -proc tableSearch(t: TTable, key, closure: RootRef, - comparator: TCmpProc): RootRef = - var h: Hash = hashNode(key) and high(t.data) # start with real hash value - while t.data[h].key != nil: - if t.data[h].key == key: - if comparator(t.data[h].val, closure): - # BUGFIX 1 - return t.data[h].val - h = nextTry(h, high(t.data)) - result = nil - -proc tableGet(t: TTable, key: RootRef): RootRef = - var index = tableRawGet(t, key) - if index >= 0: result = t.data[index].val - else: result = nil - -proc tableRawInsert(data: var TPairSeq, key, val: RootRef) = - var h: Hash = hashNode(key) and high(data) - while data[h].key != nil: - assert(data[h].key != key) - h = nextTry(h, high(data)) - assert(data[h].key == nil) - data[h].key = key - data[h].val = val - -proc tableEnlarge(t: var TTable) = - var n: TPairSeq - newSeq(n, len(t.data) * GrowthFactor) - for i in countup(0, high(t.data)): - if t.data[i].key != nil: tableRawInsert(n, t.data[i].key, t.data[i].val) - swap(t.data, n) - -proc tablePut(t: var TTable, key, val: RootRef) = - var index = tableRawGet(t, key) - if index >= 0: - t.data[index].val = val - else: - if mustRehash(len(t.data), t.counter): tableEnlarge(t) - tableRawInsert(t.data, key, val) - inc(t.counter) - proc strTableContains(t: TStrTable, n: PSym): bool = var h: Hash = n.name.h and high(t.data) # start with real hash value while t.data[h] != nil: diff --git a/compiler/canonicalizer.nim b/compiler/canonicalizer.nim index 089bce302..2abe0a0e6 100644 --- a/compiler/canonicalizer.nim +++ b/compiler/canonicalizer.nim @@ -383,18 +383,28 @@ proc createDb() = interfHash varchar(256) not null, fullHash varchar(256) not null, - created timestamp not null default (DATETIME('now')), + created timestamp not null default (DATETIME('now')) );""") db.exec(sql""" + create table if not exists Backend( + id integer primary key, + strongdeps varchar(max) not null, + weakdeps varchar(max) not null, + header varchar(max) not null, + code varchar(max) not null + ) + create table if not exists Symbol( id integer primary key, module integer not null, + backend integer not null, name varchar(max) not null, data varchar(max) not null, created timestamp not null default (DATETIME('now')), - foreign key (module) references module(id) + foreign key (module) references Module(id), + foreign key (backend) references Backend(id) );""") db.exec(sql""" @@ -409,7 +419,3 @@ proc createDb() = );""") - #db.exec(sql""" - # --create unique index if not exists TsstNameIx on TestResult(name); - # """, []) - diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index dffb8a9a5..48157925c 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -411,7 +411,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope = add(result, substr(pat, start, i - 1)) proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) = - var op, a: TLoc + var op: TLoc initLocExpr(p, ri.sons[0], op) # getUniqueType() is too expensive here: var typ = skipTypes(ri.sons[0].typ, abstractInst) @@ -458,7 +458,7 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) = proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) = # generates a crappy ObjC call - var op, a: TLoc + var op: TLoc initLocExpr(p, ri.sons[0], op) var pl = ~"[" # getUniqueType() is too expensive here: @@ -536,8 +536,6 @@ proc genCall(p: BProc, e: PNode, d: var TLoc) = else: genPrefixCall(p, nil, e, d) postStmtActions(p) - when false: - if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d) proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = if ri.sons[0].typ.skipTypes({tyGenericInst}).callConv == ccClosure: @@ -549,6 +547,3 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = else: genPrefixCall(p, le, ri, d) postStmtActions(p) - when false: - if d.s == onStack and containsGarbageCollectedRef(d.t): keepAlive(p, d) - diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 6c96f209e..be49ddc87 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -30,19 +30,6 @@ proc intLiteral(i: BiggestInt): Rope = else: result = ~"(IL64(-9223372036854775807) - IL64(1))" -proc int32Literal(i: int): Rope = - if i == int(low(int32)): - result = ~"(-2147483647 -1)" - else: - result = rope(i) - -proc genHexLiteral(v: PNode): Rope = - # hex literals are unsigned in C - # so we don't generate hex literals any longer. - if v.kind notin {nkIntLit..nkUInt64Lit}: - internalError(v.info, "genHexLiteral") - result = intLiteral(v.intVal) - proc getStrLit(m: BModule, s: string): Rope = discard cgsym(m, "TGenericSeq") result = getTempName(m) @@ -171,7 +158,6 @@ proc getStorageLoc(n: PNode): TStorageLoc = proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = if dest.s == OnStack or not usesNativeGC(): linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) - if needToKeepAlive in flags: keepAlive(p, dest) elif dest.s == OnHeap: # location is on heap # now the writer barrier is inlined for performance: @@ -198,7 +184,6 @@ proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = else: linefmt(p, cpsStmts, "#unsureAsgnRef((void**) $1, $2);$n", addrLoc(dest), rdLoc(src)) - if needToKeepAlive in flags: keepAlive(p, dest) proc asgnComplexity(n: PNode): int = if n != nil: @@ -218,7 +203,6 @@ proc optAsgnLoc(a: TLoc, t: PType, field: Rope): TLoc = result.s = a.s result.t = t result.r = rdLoc(a) & "." & field - result.heapRoot = a.heapRoot proc genOptAsgnTuple(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = let newflags = @@ -268,7 +252,6 @@ proc genGenericAsgn(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = linefmt(p, cpsStmts, "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n", addrLoc(dest), addrLoc(src), rdLoc(dest)) - if needToKeepAlive in flags: keepAlive(p, dest) else: linefmt(p, cpsStmts, "#genericShallowAssign((void*)$1, (void*)$2, $3);$n", addrLoc(dest), addrLoc(src), genTypeInfo(p.module, dest.t)) @@ -299,7 +282,6 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = else: if dest.s == OnStack or not usesNativeGC(): linefmt(p, cpsStmts, "$1 = #copyString($2);$n", dest.rdLoc, src.rdLoc) - if needToKeepAlive in flags: keepAlive(p, dest) elif dest.s == OnHeap: # we use a temporary to care for the dreaded self assignment: var tmp: TLoc @@ -310,7 +292,6 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = else: linefmt(p, cpsStmts, "#unsureAsgnRef((void**) $1, #copyString($2));$n", addrLoc(dest), rdLoc(src)) - if needToKeepAlive in flags: keepAlive(p, dest) of tyProc: if needsComplexAssignment(dest.t): # optimize closure assignment: @@ -400,9 +381,6 @@ proc genDeepCopy(p: BProc; dest, src: TLoc) = linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) else: internalError("genDeepCopy: " & $ty.kind) -proc getDestLoc(p: BProc, d: var TLoc, typ: PType) = - if d.k == locNone: getTemp(p, typ, d) - proc putLocIntoDest(p: BProc, d: var TLoc, s: TLoc) = if d.k != locNone: if lfNoDeepCopy in d.flags: genAssignment(p, d, s, {}) @@ -453,13 +431,6 @@ proc unaryStmt(p: BProc, e: PNode, d: var TLoc, frmt: string) = initLocExpr(p, e.sons[1], a) lineCg(p, cpsStmts, frmt, [rdLoc(a)]) -proc binaryStmtChar(p: BProc, e: PNode, d: var TLoc, frmt: string) = - var a, b: TLoc - if (d.k != locNone): internalError(e.info, "binaryStmtChar") - initLocExpr(p, e.sons[1], a) - initLocExpr(p, e.sons[2], b) - lineCg(p, cpsStmts, frmt, [rdCharLoc(a), rdCharLoc(b)]) - proc binaryExpr(p: BProc, e: PNode, d: var TLoc, frmt: string) = var a, b: TLoc assert(e.sons[1].typ != nil) @@ -728,8 +699,6 @@ proc genAddr(p: BProc, e: PNode, d: var TLoc) = template inheritLocation(d: var TLoc, a: TLoc) = if d.k == locNone: d.s = a.s - if d.heapRoot == nil: - d.heapRoot = if a.heapRoot != nil: a.heapRoot else: a.r proc genRecordFieldAux(p: BProc, e: PNode, d, a: var TLoc): PType = initLocExpr(p, e.sons[0], a) @@ -757,6 +726,7 @@ proc lookupFieldAgain(p: BProc, ty: PType; field: PSym; r: var Rope): PSym = var ty = ty assert r != nil while ty != nil: + ty = ty.skipTypes(skipPtrs) assert(ty.kind in {tyTuple, tyObject}) result = lookupInRecord(ty.n, field.name) if result != nil: break @@ -893,7 +863,6 @@ proc genSeqElem(p: BProc, x, y: PNode, d: var TLoc) = "if ((NU)($1) >= (NU)($2->$3)) #raiseIndexError();$n", rdLoc(b), rdLoc(a), lenField(p)) if d.k == locNone: d.s = OnHeap - d.heapRoot = a.r if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}: a.r = rfmt(nil, "(*$1)", a.r) putIntoDest(p, d, elemType(skipTypes(a.t, abstractVar)), @@ -1008,9 +977,8 @@ proc genStrConcat(p: BProc, e: PNode, d: var TLoc) = add(p.s(cpsStmts), appends) if d.k == locNone: d = tmp - keepAlive(p, tmp) else: - genAssignment(p, d, tmp, {needToKeepAlive}) # no need for deep copying + genAssignment(p, d, tmp, {}) # no need for deep copying gcUsage(e) proc genStrAppend(p: BProc, e: PNode, d: var TLoc) = @@ -1047,7 +1015,6 @@ proc genStrAppend(p: BProc, e: PNode, d: var TLoc) = rdLoc(dest), rdLoc(a))) linefmt(p, cpsStmts, "$1 = #resizeString($1, $2$3);$n", rdLoc(dest), lens, rope(L)) - keepAlive(p, dest) add(p.s(cpsStmts), appends) gcUsage(e) @@ -1067,7 +1034,6 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) = rdLoc(a), getTypeDesc(p.module, skipTypes(e.sons[1].typ, abstractVar)), getTypeDesc(p.module, bt)]) - keepAlive(p, a) #if bt != b.t: # echo "YES ", e.info, " new: ", typeToString(bt), " old: ", typeToString(b.t) initLoc(dest, locExpr, bt, OnHeap) @@ -1094,7 +1060,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) = genTypeInfo(p.module, refType), sizeExpr] if a.s == OnHeap and usesNativeGC(): - # use newObjRC1 as an optimization; and we don't need 'keepAlive' either + # use newObjRC1 as an optimization if canFormAcycle(a.t): linefmt(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc) else: @@ -1103,7 +1069,7 @@ proc rawGenNew(p: BProc, a: TLoc, sizeExpr: Rope) = linefmt(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc) else: b.r = ropecg(p.module, "($1) #newObj($2, $3)", args) - genAssignment(p, a, b, {needToKeepAlive}) # set the object type: + genAssignment(p, a, b, {}) # set the object type: let bt = skipTypes(refType.sons[0], abstractRange) genObjectInit(p, cpsStmts, bt, a, false) @@ -1134,7 +1100,7 @@ proc genNewSeqAux(p: BProc, dest: TLoc, length: Rope) = linefmt(p, cpsStmts, "$1 = $2;$n", dest.rdLoc, call.rdLoc) else: call.r = ropecg(p.module, "($1) #newSeq($2, $3)", args) - genAssignment(p, dest, call, {needToKeepAlive}) + genAssignment(p, dest, call, {}) proc genNewSeq(p: BProc, e: PNode) = var a, b: TLoc @@ -1199,7 +1165,6 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) = tmp2.k = locTemp tmp2.t = field.loc.t tmp2.s = if isRef: OnHeap else: OnStack - tmp2.heapRoot = tmp.r expr(p, it.sons[1], tmp2) if d.k == locNone: @@ -1246,7 +1211,6 @@ proc genNewFinalize(p: BProc, e: PNode) = a, b, f: TLoc refType, bt: PType ti: Rope - oldModule: BModule refType = skipTypes(e.sons[1].typ, abstractVarRange) initLocExpr(p, e.sons[1], a) initLocExpr(p, e.sons[2], f) @@ -1256,7 +1220,7 @@ proc genNewFinalize(p: BProc, e: PNode) = b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", [ getTypeDesc(p.module, refType), ti, getTypeDesc(p.module, skipTypes(refType.lastSon, abstractRange))]) - genAssignment(p, a, b, {needToKeepAlive}) # set the object type: + genAssignment(p, a, b, {}) # set the object type: bt = skipTypes(refType.lastSon, abstractRange) genObjectInit(p, cpsStmts, bt, a, false) gcUsage(e) @@ -1294,7 +1258,7 @@ proc genOf(p: BProc, x: PNode, typ: PType, d: var TLoc) = if not p.module.compileToCpp: while t.kind == tyObject and t.sons[0] != nil: add(r, ~".Sup") - t = skipTypes(t.sons[0], typedescInst) + t = skipTypes(t.sons[0], skipPtrs) if isObjLackingTypeField(t): globalError(x.info, errGenerated, "no 'of' operator available for pure objects") @@ -1366,7 +1330,7 @@ proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = initLocExpr(p, n.sons[1], a) a.r = ropecg(p.module, frmt, [rdLoc(a)]) if d.k == locNone: getTemp(p, n.typ, d) - genAssignment(p, d, a, {needToKeepAlive}) + genAssignment(p, d, a, {}) gcUsage(n) proc genArrayLen(p: BProc, e: PNode, d: var TLoc, op: TMagic) = @@ -1408,12 +1372,10 @@ proc genSetLengthSeq(p: BProc, e: PNode, d: var TLoc) = lineCg(p, cpsStmts, setLenPattern, [ rdLoc(a), rdLoc(b), getTypeDesc(p.module, t), getTypeDesc(p.module, t.sons[0])]) - keepAlive(p, a) gcUsage(e) proc genSetLengthStr(p: BProc, e: PNode, d: var TLoc) = binaryStmt(p, e, d, "$1 = #setLengthStr($1, $2);$n") - keepAlive(p, d) gcUsage(e) proc genSwap(p: BProc, e: PNode, d: var TLoc) = @@ -1683,7 +1645,6 @@ proc binaryFloatArith(p: BProc, e: PNode, d: var TLoc, m: TMagic) = binaryArith(p, e, d, m) proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = - var line, filen: Rope case op of mOr, mAnd: genAndOr(p, e, d, op) of mNot..mToBiggestInt: unaryArith(p, e, d, op) @@ -1721,10 +1682,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = getTypeDesc(p.module, ranged), res]) of mConStrStr: genStrConcat(p, e, d) - of mAppendStrCh: - binaryStmt(p, e, d, "$1 = #addChar($1, $2);$n") - # strictly speaking we need to generate "keepAlive" here too, but this - # very likely not needed and would slow down the code too much I fear + of mAppendStrCh: binaryStmt(p, e, d, "$1 = #addChar($1, $2);$n") of mAppendStrStr: genStrAppend(p, e, d) of mAppendSeqElem: genSeqElemAppend(p, e, d) of mEqStr: genStrEquals(p, e, d) @@ -1918,7 +1876,7 @@ proc upConv(p: BProc, n: PNode, d: var TLoc) = if not p.module.compileToCpp: while t.kind == tyObject and t.sons[0] != nil: add(r, ".Sup") - t = skipTypes(t.sons[0], abstractInst) + t = skipTypes(t.sons[0], skipPtrs) if nilCheck != nil: linefmt(p, cpsStmts, "if ($1) #chckObj($2.m_type, $3);$n", nilCheck, r, genTypeInfo(p.module, dest)) @@ -2018,7 +1976,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of skEnumField: putIntoDest(p, d, n.typ, rope(sym.position)) of skVar, skForVar, skResult, skLet: - if sfGlobal in sym.flags: genVarPrototype(p.module, sym) + if {sfGlobal, sfThread} * sym.flags != {}: + genVarPrototype(p.module, sym) if sym.loc.r == nil or sym.loc.t == nil: #echo "FAILED FOR PRCO ", p.prc.name.s #echo renderTree(p.prc.ast, {renderIds}) @@ -2126,11 +2085,14 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkCaseStmt: genCase(p, n, d) of nkReturnStmt: genReturnStmt(p, n) of nkBreakStmt: genBreakStmt(p, n) - of nkAsgn: genAsgn(p, n, fastAsgn=false) + of nkAsgn: + if nfPreventCg notin n.flags: + genAsgn(p, n, fastAsgn=false) of nkFastAsgn: - # transf is overly aggressive with 'nkFastAsgn', so we work around here. - # See tests/run/tcnstseq3 for an example that would fail otherwise. - genAsgn(p, n, fastAsgn=p.prc != nil) + if nfPreventCg notin n.flags: + # transf is overly aggressive with 'nkFastAsgn', so we work around here. + # See tests/run/tcnstseq3 for an example that would fail otherwise. + genAsgn(p, n, fastAsgn=p.prc != nil) of nkDiscardStmt: if n.sons[0].kind != nkEmpty: genLineDir(p, n) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index a5ce147c3..10b5641c5 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -21,7 +21,7 @@ proc registerGcRoot(p: BProc, v: PSym) = # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) let prc = genTraverseProcForGlobal(p.module, v) - appcg(p.module, p.module.initProc.procSec(cpsStmts), + appcg(p.module, p.module.initProc.procSec(cpsInit), "#nimRegisterGlobalMarker($1);$n", [prc]) proc isAssignedImmediately(n: PNode): bool {.inline.} = @@ -293,6 +293,8 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) = genLineDir(p, n) let lend = getLabel(p) for i in countup(0, sonsLen(n) - 1): + # bug #4230: avoid false sharing between branches: + if d.k == locTemp and isEmptyType(n.typ): d.k = locNone let it = n.sons[i] if it.len == 2: when newScopeForIf: startBlock(p) @@ -359,6 +361,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) = linefmt(p, cpsStmts, "#popCurrentException();$n") proc genReturnStmt(p: BProc, t: PNode) = + if nfPreventCg in t.flags: return p.beforeRetNeeded = true genLineDir(p, t) if (t.sons[0].kind != nkEmpty): genStmts(p, t.sons[0]) @@ -459,7 +462,6 @@ proc genWhileStmt(p: BProc, t: PNode) = # significantly worse code var a: TLoc - labl: TLabel assert(sonsLen(t) == 2) inc(p.withinLoop) genLineDir(p, t) @@ -488,16 +490,20 @@ proc genWhileStmt(p: BProc, t: PNode) = dec(p.withinLoop) -proc genBlock(p: BProc, t: PNode, d: var TLoc) = +proc genBlock(p: BProc, n: PNode, d: var TLoc) = + # bug #4505: allocate the temp in the outer scope + # so that it can escape the generated {}: + if not isEmptyType(n.typ) and d.k == locNone: + getTemp(p, n.typ, d) preserveBreakIdx: p.breakIdx = startBlock(p) - if t.sons[0].kind != nkEmpty: + if n.sons[0].kind != nkEmpty: # named block? - assert(t.sons[0].kind == nkSym) - var sym = t.sons[0].sym + assert(n.sons[0].kind == nkSym) + var sym = n.sons[0].sym sym.loc.k = locOther sym.position = p.breakIdx+1 - expr(p, t.sons[1], d) + expr(p, n.sons[1], d) endBlock(p) proc genParForStmt(p: BProc, t: PNode) = @@ -592,6 +598,8 @@ proc genCaseSecondPass(p: BProc, t: PNode, d: var TLoc, labId, until: int): TLabel = var lend = getLabel(p) for i in 1..until: + # bug #4230: avoid false sharing between branches: + if d.k == locTemp and isEmptyType(t.typ): d.k = locNone lineF(p, cpsStmts, "LA$1: ;$n", [rope(labId + i)]) if t.sons[i].kind == nkOfBranch: var length = sonsLen(t.sons[i]) @@ -727,6 +735,8 @@ proc genOrdinalCase(p: BProc, n: PNode, d: var TLoc) = lineF(p, cpsStmts, "switch ($1) {$n", [rdCharLoc(a)]) var hasDefault = false for i in splitPoint+1 .. < n.len: + # bug #4230: avoid false sharing between branches: + if d.k == locTemp and isEmptyType(n.typ): d.k = locNone var branch = n[i] if branch.kind == nkOfBranch: genCaseRange(p, branch) @@ -757,16 +767,6 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) = else: genOrdinalCase(p, t, d) -proc hasGeneralExceptSection(t: PNode): bool = - var length = sonsLen(t) - var i = 1 - while (i < length) and (t.sons[i].kind == nkExceptBranch): - var blen = sonsLen(t.sons[i]) - if blen == 1: - return true - inc(i) - result = false - proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = # code to generate: # @@ -807,6 +807,8 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = var i = 1 var catchAllPresent = false while (i < length) and (t.sons[i].kind == nkExceptBranch): + # bug #4230: avoid false sharing between branches: + if d.k == locTemp and isEmptyType(t.typ): d.k = locNone let blen = sonsLen(t.sons[i]) if i > 1: addf(p.s(cpsStmts), "else ", []) if blen == 1: @@ -912,6 +914,8 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) = inc p.inExceptBlock var i = 1 while (i < length) and (t.sons[i].kind == nkExceptBranch): + # bug #4230: avoid false sharing between branches: + if d.k == locTemp and isEmptyType(t.typ): d.k = locNone var blen = sonsLen(t.sons[i]) if blen == 1: # general except section: @@ -1089,7 +1093,6 @@ proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType, proc asgnFieldDiscriminant(p: BProc, e: PNode) = var a, tmp: TLoc var dotExpr = e.sons[0] - var d: PSym if dotExpr.kind == nkCheckedFieldExpr: dotExpr = dotExpr.sons[0] initLocExpr(p, e.sons[0], a) getTemp(p, a.t, tmp) diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index c4d47f148..a8c079b35 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -73,7 +73,9 @@ proc genTraverseProc(c: var TTraversalClosure, accessor: Rope, typ: PType) = lineF(p, cpsStmts, "}$n", []) of tyObject: for i in countup(0, sonsLen(typ) - 1): - genTraverseProc(c, accessor.parentObj(c.p.module), typ.sons[i]) + var x = typ.sons[i] + if x != nil: x = x.skipTypes(skipPtrs) + genTraverseProc(c, accessor.parentObj(c.p.module), x) if typ.n != nil: genTraverseProc(c, accessor, typ.n) of tyTuple: let typ = getUniqueType(typ) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 3d1c2affc..eac734b3d 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -475,11 +475,11 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, hasField = true elif m.compileToCpp: appcg(m, result, " : public $1 {$n", - [getTypeDescAux(m, typ.sons[0], check)]) + [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)]) hasField = true else: appcg(m, result, " {$n $1 Sup;$n", - [getTypeDescAux(m, typ.sons[0], check)]) + [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)]) hasField = true else: addf(result, " {$n", [name]) @@ -546,7 +546,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = of tyRef, tyPtr, tyVar: var star = if t.kind == tyVar and tfVarIsPtr notin typ.flags and compileToCpp(m): "&" else: "*" - var et = t.lastSon + var et = typ.skipTypes(abstractInst).lastSon var etB = et.skipTypes(abstractInst) if etB.kind in {tyArrayConstr, tyArray, tyOpenArray, tyVarargs}: # this is correct! sets have no proper base type, so we treat @@ -744,14 +744,6 @@ proc getClosureType(m: BModule, t: PType, kind: TClosureTypeKind): Rope = "void* ClEnv;$n} $1;$n", [result, rettype, desc]) -proc getTypeDesc(m: BModule, magic: string): Rope = - var sym = magicsys.getCompilerProc(magic) - if sym != nil: - result = getTypeDesc(m, sym.typ) - else: - rawMessage(errSystemNeeds, magic) - result = nil - proc finishTypeDescriptions(m: BModule) = var i = 0 while i < len(m.typeStack): @@ -820,7 +812,9 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope) = proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope) = var base: Rope if (sonsLen(typ) > 0) and (typ.sons[0] != nil): - base = genTypeInfo(m, typ.sons[0]) + var x = typ.sons[0] + if typ.kind == tyObject: x = x.skipTypes(skipPtrs) + base = genTypeInfo(m, x) else: base = rope("0") genTypeInfoAuxBase(m, typ, origType, name, base) @@ -894,10 +888,11 @@ proc genObjectFields(m: BModule, typ: PType, n: PNode, expr: Rope) = else: internalError(n.info, "genObjectFields(nkRecCase)") of nkSym: var field = n.sym - addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" & - "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" & - "$1.name = $5;$n", [expr, getTypeDesc(m, typ), - field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)]) + if field.bitsize == 0: + addf(m.s[cfsTypeInit3], "$1.kind = 1;$n" & + "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" & + "$1.name = $5;$n", [expr, getTypeDesc(m, typ), + field.loc.r, genTypeInfo(m, field.typ), makeCString(field.name.s)]) else: internalError(n.info, "genObjectFields") proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) = @@ -909,7 +904,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope) = addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp]) var t = typ.sons[0] while t != nil: - t = t.skipTypes(abstractInst) + t = t.skipTypes(skipPtrs) t.flags.incl tfObjHasKids t = t.sons[0] @@ -1000,9 +995,6 @@ proc fakeClosureType(owner: PSym): PType = type TTypeInfoReason = enum ## for what do we need the type info? tiNew, ## for 'new' - tiNewSeq, ## for 'newSeq' - tiNonVariantAsgn, ## for generic assignment without variants - tiVariantAsgn ## for generic assignment with variants include ccgtrav diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 9851ab0e2..d80a68609 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -167,10 +167,6 @@ proc linefmt(p: BProc, s: TCProcSection, frmt: FormatStr, args: varargs[Rope]) = add(p.s(s), indentLine(p, ropecg(p.module, frmt, args))) -proc appLineCg(p: BProc, r: var Rope, frmt: FormatStr, - args: varargs[Rope]) = - add(r, indentLine(p, ropecg(p.module, frmt, args))) - proc safeLineNm(info: TLineInfo): int = result = toLinenumber(info) if result < 0: result = 0 # negative numbers are not allowed in #line @@ -250,7 +246,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc, if not p.module.compileToCpp: while (s.kind == tyObject) and (s.sons[0] != nil): add(r, ".Sup") - s = skipTypes(s.sons[0], abstractInst) + s = skipTypes(s.sons[0], skipPtrs) linefmt(p, section, "$1.m_type = $2;$n", r, genTypeInfo(p.module, t)) of frEmbedded: # worst case for performance: @@ -259,8 +255,7 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: TLoc, type TAssignmentFlag = enum - needToCopy, needForSubtypeCheck, afDestIsNil, afDestIsNotNil, afSrcIsNil, - afSrcIsNotNil, needToKeepAlive + needToCopy, afDestIsNil, afDestIsNotNil, afSrcIsNil, afSrcIsNotNil TAssignmentFlags = set[TAssignmentFlag] proc genRefAssign(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) @@ -328,8 +323,9 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) = proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = inc(p.labels) result.r = "LOC" & rope(p.labels) - addf(p.blocks[0].sections[cpsLocals], - "$1 $2;$n", [getTypeDesc(p.module, t), result.r]) + #addf(p.blocks[0].sections[cpsLocals], + # "$1 $2;$n", [getTypeDesc(p.module, t), result.r]) + linefmt(p, cpsLocals, "$1 $2;$n", getTypeDesc(p.module, t), result.r) result.k = locTemp #result.a = - 1 result.t = t @@ -338,31 +334,6 @@ proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) = result.flags = {} constructLoc(p, result, not needsInit) -proc keepAlive(p: BProc, toKeepAlive: TLoc) = - when false: - # deactivated because of the huge slowdown this causes; GC will take care - # of interior pointers instead - if optRefcGC notin gGlobalOptions: return - var result: TLoc - var fid = rope(p.gcFrameId) - result.r = "GCFRAME.F" & fid - addf(p.gcFrameType, " $1 F$2;$n", - [getTypeDesc(p.module, toKeepAlive.t), fid]) - inc(p.gcFrameId) - result.k = locTemp - #result.a = -1 - result.t = toKeepAlive.t - result.s = OnStack - result.flags = {} - - if not isComplexValueType(skipTypes(toKeepAlive.t, abstractVarRange)): - linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(result), rdLoc(toKeepAlive)) - else: - useStringh(p.module) - linefmt(p, cpsStmts, - "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($3));$n", - addrLoc(result), addrLoc(toKeepAlive), rdLoc(result)) - proc initGCFrame(p: BProc): Rope = if p.gcFrameId > 0: result = "struct {$1} GCFRAME;$n" % [p.gcFrameType] @@ -620,9 +591,6 @@ proc generateHeaders(m: BModule) = addf(m.s[cfsHeaders], "#include $1$N", [rope(it.data)]) it = PStrEntry(it.next) -proc retIsNotVoid(s: PSym): bool = - result = (s.typ.sons[0] != nil) and not isInvalidReturnType(s.typ.sons[0]) - proc initFrame(p: BProc, procname, filename: Rope): Rope = discard cgsym(p.module, "nimFrame") if p.maxFrameLen > 0: @@ -649,6 +617,24 @@ proc closureSetup(p: BProc, prc: PSym) = linefmt(p, cpsStmts, "$1 = ($2) ClEnv;$n", rdLoc(env.loc), getTypeDesc(p.module, env.typ)) +proc easyResultAsgn(n: PNode): PNode = + const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt} + + declarativeDefs + case n.kind + of nkStmtList, nkStmtListExpr: + var i = 0 + while i < n.len and n[i].kind in harmless: inc i + if i < n.len: result = easyResultAsgn(n[i]) + of nkAsgn, nkFastAsgn: + if n[0].kind == nkSym and skResult == n[0].sym.kind: + incl n.flags, nfPreventCg + return n[1] + of nkReturnStmt: + if n.len > 0: + result = easyResultAsgn(n[0]) + if result != nil: incl n.flags, nfPreventCg + else: discard + proc genProcAux(m: BModule, prc: PSym) = var p = newProc(prc, m) var header = genProcHeader(m, prc) @@ -660,11 +646,17 @@ proc genProcAux(m: BModule, prc: PSym) = var res = prc.ast.sons[resultPos].sym # get result symbol if not isInvalidReturnType(prc.typ.sons[0]): if sfNoInit in prc.flags: incl(res.flags, sfNoInit) - # declare the result symbol: - assignLocalVar(p, res) - assert(res.loc.r != nil) + if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(prc.getBody); val != nil): + var decl = localVarDecl(p, res) + var a: TLoc + initLocExprSingleUse(p, val, a) + linefmt(p, cpsStmts, "$1 = $2;$n", decl, rdLoc(a)) + else: + # declare the result symbol: + assignLocalVar(p, res) + assert(res.loc.r != nil) + initLocalVar(p, res, immediateAsgn=false) returnStmt = rfmt(nil, "\treturn $1;$n", rdLoc(res.loc)) - initLocalVar(p, res, immediateAsgn=false) else: fillResult(res) assignParam(p, res) @@ -796,7 +788,7 @@ proc genProc(m: BModule, prc: PSym) = genProcAux(generatedHeader, prc) proc genVarPrototypeAux(m: BModule, sym: PSym) = - assert(sfGlobal in sym.flags) + #assert(sfGlobal in sym.flags) useHeader(m, sym) fillLoc(sym.loc, locGlobalVar, sym.typ, mangleName(sym), OnHeap) if (lfNoDecl in sym.loc.flags) or containsOrIncl(m.declaredThings, sym.id): @@ -824,12 +816,12 @@ proc addIntTypes(result: var Rope) {.inline.} = proc getCopyright(cfile: string): Rope = if optCompileOnly in gGlobalOptions: result = ("/* Generated by Nim Compiler v$1 */$N" & - "/* (c) 2015 Andreas Rumpf */$N" & + "/* (c) 2016 Andreas Rumpf */$N" & "/* The generated code is subject to the original license. */$N") % [rope(VersionAsString)] else: result = ("/* Generated by Nim Compiler v$1 */$N" & - "/* (c) 2015 Andreas Rumpf */$N" & + "/* (c) 2016 Andreas Rumpf */$N" & "/* The generated code is subject to the original license. */$N" & "/* Compiled for: $2, $3, $4 */$N" & "/* Command for C compiler:$n $5 */$N") % @@ -875,10 +867,11 @@ proc genMainProc(m: BModule) = MainProcsWithResult = MainProcs & "\treturn nim_program_result;$N" - NimMainBody = - "N_CDECL(void, NimMainInner)(void) {$N" & + NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" & "$1" & - "}$N$N" & + "}$N$N" + + NimMainProc = "N_CDECL(void, NimMain)(void) {$N" & "\tvoid (*volatile inner)();$N" & "\tPreMain();$N" & @@ -887,6 +880,8 @@ proc genMainProc(m: BModule) = "\t(*inner)();$N" & "}$N$N" + NimMainBody = NimMainInner & NimMainProc + PosixNimMain = "int cmdCount;$N" & "char** cmdLine;$N" & @@ -914,7 +909,7 @@ proc genMainProc(m: BModule) = " LPSTR lpCmdLine, int nCmdShow) {$N" & MainProcsWithResult & "}$N$N" - WinNimDllMain = "N_LIB_EXPORT " & NimMainBody + WinNimDllMain = NimMainInner & "N_LIB_EXPORT " & NimMainProc WinCDllMain = "BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdreason, $N" & @@ -1020,7 +1015,7 @@ proc genInitCode(m: BModule) = var procname = makeCString(m.module.name.s) add(prc, initFrame(m.initProc, procname, m.module.info.quotedFilename)) else: - add(prc, ~"\tTFrame F; FR.len = 0;$N") + add(prc, ~"\tTFrame FR; FR.len = 0;$N") add(prc, genSectionStart(cpsInit)) add(prc, m.preInitProc.s(cpsInit)) diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 624c5b183..bcf0b535b 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -145,15 +145,16 @@ proc fixupDispatcher(meth, disp: PSym) = disp.typ.lockLevel = meth.typ.lockLevel proc methodDef*(s: PSym, fromCache: bool) = - var L = len(gMethods) + let L = len(gMethods) var witness: PSym for i in countup(0, L - 1): - var disp = gMethods[i].dispatcher + let disp = gMethods[i].dispatcher case sameMethodBucket(disp, s) of Yes: add(gMethods[i].methods, s) attachDispatcher(s, lastSon(disp.ast)) fixupDispatcher(s, disp) + #echo "fixup ", disp.name.s, " ", disp.id when useEffectSystem: checkMethodEffects(disp, s) if sfBase in s.flags and gMethods[i].methods[0] != s: # already exists due to forwarding definition? diff --git a/compiler/docgen.nim b/compiler/docgen.nim index bbbec081a..c220902ff 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -27,6 +27,7 @@ type seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML. jArray: JsonNode types: TStrTable + isPureRst: bool PDoc* = ref TDocumentor ## Alias to type less. @@ -440,11 +441,6 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = dispA(result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}", [rope(esc(d.target, literal))]) - if k in routineKinds and nameNode.kind == nkSym: - let att = attachToType(d, nameNode.sym) - if att != nil: - dispA(result, """<span class="attachedType" style="visibility:hidden">$1</span>""", "", - [rope esc(d.target, att.name.s)]) inc(d.id) let plainNameRope = rope(xmltree.escape(plainName.strip)) @@ -459,25 +455,35 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) = var seeSrcRope: Rope = nil let docItemSeeSrc = getConfigVar("doc.item.seesrc") - if docItemSeeSrc.len > 0 and options.docSeeSrcUrl.len > 0: - let path = n.info.toFilename.extractFilename.rope - let urlRope = ropeFormatNamedVars(options.docSeeSrcUrl, - ["path", "line"], [path, rope($n.info.line)]) + if docItemSeeSrc.len > 0: + let cwd = getCurrentDir().canonicalizePath() + var path = n.info.toFullPath + if path.startsWith(cwd): + path = path[cwd.len+1 .. ^1].replace('\\', '/') + var commit = getConfigVar("git.commit") + if commit.len == 0: commit = "master" dispA(seeSrcRope, "$1", "", [ropeFormatNamedVars(docItemSeeSrc, - ["path", "line", "url"], [path, - rope($n.info.line), urlRope])]) + ["path", "line", "url", "commit"], [rope path, + rope($n.info.line), rope getConfigVar("git.url"), + rope commit])]) add(d.section[k], ropeFormatNamedVars(getConfigVar("doc.item"), ["name", "header", "desc", "itemID", "header_plain", "itemSym", "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "seeSrc"], [nameRope, result, comm, itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope])) + + var attype: Rope + if k in routineKinds and nameNode.kind == nkSym: + let att = attachToType(d, nameNode.sym) + if att != nil: + attype = rope esc(d.target, att.name.s) add(d.toc[k], ropeFormatNamedVars(getConfigVar("doc.item.toc"), ["name", "header", "desc", "itemID", "header_plain", "itemSym", - "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc"], + "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "attype"], [rope(getName(d, nameNode, d.splitAfter)), result, comm, itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope, - plainSymbolEncRope, symbolOrIdEncRope])) + plainSymbolEncRope, symbolOrIdEncRope, attype])) # Ironically for types the complexSymbol is *cleaner* than the plainName # because it doesn't include object fields or documentation comments. So we @@ -629,7 +635,9 @@ proc genOutFile(d: PDoc): Rope = # Modules get an automatic title for the HTML, but no entry in the index. title = "Module " & extractFilename(changeFileExt(d.filename, "")) - let bodyname = if d.hasToc: "doc.body_toc" else: "doc.body_no_toc" + let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group" + elif d.hasToc: "doc.body_toc" + else: "doc.body_no_toc" content = ropeFormatNamedVars(getConfigVar(bodyname), ["title", "tableofcontents", "moduledesc", "date", "time", "content"], [title.rope, toc, d.modDesc, rope(getDateStr()), @@ -694,6 +702,7 @@ proc commandDoc*() = proc commandRstAux(filename, outExt: string) = var filen = addFileExt(filename, "txt") var d = newDocumentor(filen, options.gConfigVars) + d.isPureRst = true var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc, {roSupportRawDirective}) var modDesc = newStringOfCap(30_000) diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2dcfc0226..6f8b0b197 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -505,10 +505,6 @@ proc noAbsolutePaths: bool {.inline.} = # `optGenMapping` is included here for niminst. result = gGlobalOptions * {optGenScript, optGenMapping} != {} -const - specialFileA = 42 - specialFileB = 42 - var fileCounter: int proc add(s: var string, many: openArray[string]) = diff --git a/compiler/idgen.nim b/compiler/idgen.nim index 333772705..c6b1a4d07 100644 --- a/compiler/idgen.nim +++ b/compiler/idgen.nim @@ -54,6 +54,6 @@ proc loadMaxIds*(project: string) = if f.readLine(line): var frontEndId = parseInt(line) if f.readLine(line): - var backEndId = parseInt(line) + # var backEndId = parseInt(line) gFrontEndId = max(gFrontEndId, frontEndId) f.close() diff --git a/compiler/importer.nim b/compiler/importer.nim index dd2c4d954..87415733b 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -163,7 +163,10 @@ proc myImportModule(c: PContext, n: PNode): PSym = var f = checkModuleName(n) if f != InvalidFileIDX: result = importModuleAs(n, gImportModule(c.module, f)) - if result.info.fileIndex == n.info.fileIndex: + # we cannot perform this check reliably because of + # test: modules/import_in_config) + if result.info.fileIndex == c.module.info.fileIndex and + result.info.fileIndex == n.info.fileIndex: localError(n.info, errGenerated, "A module cannot import itself") if sfDeprecated in result.flags: message(n.info, warnDeprecated, result.name.s) diff --git a/compiler/installer.ini b/compiler/installer.ini index b802f08f1..acc82002d 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -50,6 +50,7 @@ Files: "readme.txt;install.txt;contributors.txt;copying.txt" Files: "makefile" Files: "koch.nim" Files: "install_nimble.nims" +Files: "install_tools.nims" Files: "icons/nim.ico" Files: "icons/nim.rc" @@ -77,7 +78,8 @@ Files: "lib" [Other] Files: "examples" -#Files: "dist/nimble" +Files: "dist/nimble" +Files: "dist/nimsuggest" Files: "tests" @@ -87,10 +89,11 @@ Files: "bin/c2nim.exe" Files: "bin/nimgrep.exe" Files: "bin/nimsuggest.exe" Files: "bin/nimble.exe" +Files: "bin/makelink.exe" Files: "bin/*.dll" -Files: "dist/*.dll" Files: "koch.exe" +Files: "finish.exe" ; Files: "dist/mingw" Files: "start.bat" BinPath: r"bin;dist\mingw\bin;dist" @@ -98,8 +101,8 @@ BinPath: r"bin;dist\mingw\bin;dist" ; Section | dir | zipFile | size hint (in KB) | url | exe start menu entry Download: r"Documentation|doc|docs.zip|13824|http://nim-lang.org/download/docs-${version}.zip|overview.html" Download: r"C Compiler (MingW)|dist|mingw.zip|82944|http://nim-lang.org/download/${mingw}.zip" -Download: r"Support DLL's|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" -Download: r"Aporia IDE|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" +Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls.zip" +Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads [UnixBin] @@ -116,7 +119,6 @@ path = r"c:\Program Files (x86)\Inno Setup 5\iscc.exe" flags = "/Q" [NSIS] -path = r"c:\Program Files (x86)\NSIS\makensis.exe" flags = "/V0" [C_Compiler] diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index 80bcd2b0e..e7fe8cc27 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -512,6 +512,10 @@ proc arith(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = arithAux(p, n, r, op, jsOps) r.kind = resExpr +proc hasFrameInfo(p: PProc): bool = + ({optLineTrace, optStackTrace} * p.options == {optLineTrace, optStackTrace}) and + ((p.prc == nil) or not (sfPure in p.prc.flags)) + proc genLineDir(p: PProc, n: PNode) = let line = toLinenumber(n.info) if optLineDir in p.options: @@ -521,9 +525,7 @@ proc genLineDir(p: PProc, n: PNode) = ((p.prc == nil) or sfPure notin p.prc.flags): useMagic(p, "endb") addf(p.body, "endb($1);$n", [rope(line)]) - elif ({optLineTrace, optStackTrace} * p.options == - {optLineTrace, optStackTrace}) and - ((p.prc == nil) or not (sfPure in p.prc.flags)): + elif hasFrameInfo(p): addf(p.body, "F.line = $1;$n" | "$$F['line'] = $1;$n", [rope(line)]) proc genWhileStmt(p: PProc, n: PNode) = @@ -558,10 +560,13 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = # code to generate: # # ++excHandler; + # var tmpFramePtr = framePtr; # try { # stmts; + # --excHandler; # } catch (EXC) { # var prevJSError = lastJSError; lastJSError = EXC; + # framePtr = tmpFramePtr; # --excHandler; # if (e.typ && e.typ == NTI433 || e.typ == NTI2321) { # stmts; @@ -572,6 +577,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = # } # lastJSError = prevJSError; # } finally { + # framePtr = tmpFramePtr; # stmts; # } genLineDir(p, n) @@ -584,8 +590,10 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = var catchBranchesExist = length > 1 and n.sons[i].kind == nkExceptBranch if catchBranchesExist and p.target == targetJS: add(p.body, "++excHandler;" & tnl) - var safePoint = "Tmp$1" % [rope(p.unique)] - if optStackTrace in p.options: add(p.body, "framePtr = F;" & tnl) + var tmpFramePtr = rope"F" + if optStackTrace notin p.options: + tmpFramePtr = p.getTemp(true) + add(p.body, tmpFramePtr & " = framePtr;" & tnl) addf(p.body, "try {$n", []) if p.target == targetPHP and p.globals == nil: p.globals = "global $lastJSError; global $prevJSError;".rope @@ -595,8 +603,9 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = var generalCatchBranchExists = false let dollar = rope(if p.target == targetJS: "" else: "$") if p.target == targetJS and catchBranchesExist: - addf(p.body, "} catch (EXC) {$n var prevJSError = lastJSError;$n" & + addf(p.body, "--excHandler;$n} catch (EXC) {$n var prevJSError = lastJSError;$n" & " lastJSError = EXC;$n --excHandler;$n", []) + add(p.body, "framePtr = $1;$n" % [tmpFramePtr]) elif p.target == targetPHP: addf(p.body, "} catch (Exception $$EXC) {$n $$prevJSError = $$lastJSError;$n $$lastJSError = $$EXC;$n", []) while i < length and n.sons[i].kind == nkExceptBranch: @@ -618,8 +627,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = addf(orExpr, "isObj($2lastJSError.m_type, $1)", [genTypeInfo(p, n.sons[i].sons[j].typ), dollar]) if i > 1: add(p.body, "else ") - addf(p.body, "if ($3lastJSError && ($2)) {$n", - [safePoint, orExpr, dollar]) + addf(p.body, "if ($1lastJSError && ($2)) {$n", [dollar, orExpr]) gen(p, n.sons[i].sons[blen - 1], a) moveInto(p, a, r) addf(p.body, "}$n", []) @@ -631,6 +639,7 @@ proc genTry(p: PProc, n: PNode, r: var TCompRes) = addf(p.body, "$1lastJSError = $1prevJSError;$n", [dollar]) if p.target == targetJS: add(p.body, "} finally {" & tnl) + add(p.body, "framePtr = $1;$n" % [tmpFramePtr]) if p.target == targetPHP: # XXX ugly hack for PHP codegen add(p.body, "}" & tnl) @@ -1355,7 +1364,7 @@ proc createObjInitList(p: PProc, typ: PType, excludedFieldIDs: IntSet, output: v if output.len > 0: output.add(", ") addf(output, "m_type: $1" | "'m_type' => $#", [genTypeInfo(p, t)]) while t != nil: - createRecordVarAux(p, t.n, excludedFieldIDs, output) + createRecordVarAux(p, t.skipTypes(skipPtrs).n, excludedFieldIDs, output) t = t.sons[0] proc arrayTypeForElemType(typ: PType): string = @@ -1462,13 +1471,21 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = useMagic(p, "nimCopy") s = "nimCopy(null, $1, $2)" % [a.res, genTypeInfo(p, n.typ)] of etyBaseIndex: - if (a.typ != etyBaseIndex): internalError(n.info, "genVarInit") - if {sfAddrTaken, sfGlobal} * v.flags != {}: - addf(p.body, "var $1 = [$2, $3];$n", - [v.loc.r, a.address, a.res]) + let targetBaseIndex = {sfAddrTaken, sfGlobal} * v.flags == {} + if a.typ == etyBaseIndex: + if targetBaseIndex: + addf(p.body, "var $1 = $2, $1_Idx = $3;$n", [ + v.loc.r, a.address, a.res]) + else: + addf(p.body, "var $1 = [$2, $3];$n", + [v.loc.r, a.address, a.res]) else: - addf(p.body, "var $1 = $2; var $1_Idx = $3;$n", [ - v.loc.r, a.address, a.res]) + if targetBaseIndex: + let tmp = p.getTemp + addf(p.body, "var $1 = $2, $3 = $1[0], $3_Idx = $1[1];$n", + [tmp, a.res, v.loc.r]) + else: + addf(p.body, "var $1 = $2;$n", [v.loc.r, a.res]) return else: s = a.res @@ -1918,10 +1935,10 @@ proc frameCreate(p: PProc; procname, filename: Rope): Rope = procname, filename] proc frameDestroy(p: PProc): Rope = - result = rope(("framePtr = framePtr.prev;" | "$framePtr = $framePtr['prev'];") & tnl) + result = rope(("framePtr = F.prev;" | "$framePtr = $F['prev'];") & tnl) proc genProcBody(p: PProc, prc: PSym): Rope = - if optStackTrace in prc.options: + if hasFrameInfo(p): result = frameCreate(p, makeJSString(prc.owner.name.s & '.' & prc.name.s), makeJSString(toFilename(prc.info))) @@ -1935,7 +1952,7 @@ proc genProcBody(p: PProc, prc: PSym): Rope = if prc.typ.callConv == ccSysCall and p.target == targetJS: result = ("try {$n$1} catch (e) {$n" & " alert(\"Unhandled exception:\\n\" + e.message + \"\\n\"$n}") % [result] - if optStackTrace in prc.options: + if hasFrameInfo(p): add(result, frameDestroy(p)) proc genProc(oldProc: PProc, prc: PSym): Rope = diff --git a/compiler/jstypes.nim b/compiler/jstypes.nim index cf1679ee4..0aaf93579 100644 --- a/compiler/jstypes.nim +++ b/compiler/jstypes.nim @@ -76,7 +76,7 @@ proc genObjectInfo(p: PProc, typ: PType, name: Rope) = addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)]) if (typ.kind == tyObject) and (typ.sons[0] != nil): addf(p.g.typeInfo, "$1.base = $2;$n", - [name, genTypeInfo(p, typ.sons[0])]) + [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))]) proc genTupleFields(p: PProc, typ: PType): Rope = var s: Rope = nil diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 753602c80..36ad2e0a6 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -226,8 +226,14 @@ proc interestingIterVar(s: PSym): bool {.inline.} = template isIterator*(owner: PSym): bool = owner.kind == skIterator and owner.typ.callConv == ccClosure +proc liftingHarmful(owner: PSym): bool {.inline.} = + ## lambda lifting can be harmful for JS-like code generators. + let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro + result = gCmd in {cmdCompileToPHP, cmdCompileToJS} and not isCompileTime + proc liftIterSym*(n: PNode; owner: PSym): PNode = # transforms (iter) to (let env = newClosure[iter](); (iter, env)) + if liftingHarmful(owner): return n let iter = n.sym assert iter.isIterator @@ -838,6 +844,7 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode = nkBreakState(cl.state) ... """ + if liftingHarmful(owner): return body var L = body.len if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds): localError(body.info, "ignored invalid for loop") diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 0eee5004e..9c513034b 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -211,9 +211,6 @@ proc closeLexer*(lex: var TLexer) = inc(gLinesCompiled, lex.lineNumber) closeBaseLexer(lex) -proc getColumn(L: TLexer): int = - result = getColNumber(L, L.bufpos) - proc getLineInfo(L: TLexer): TLineInfo = result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos)) @@ -237,12 +234,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") = proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool = result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second) -proc isFloatLiteral(s: string): bool = - for i in countup(0, len(s) - 1): - if s[i] in {'.', 'e', 'E'}: - return true - result = false - {.push overflowChecks: off.} # We need to parse the largest uint literal without overflow checks proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int = diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 36eb24653..9db4383f6 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -172,7 +172,7 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo): PNode = if field != nil: break t = t.sons[0] if t == nil: break - t = t.skipTypes(abstractInst) + t = t.skipTypes(skipPtrs) #if field == nil: # echo "FIELD ", b # debug deref.typ @@ -193,7 +193,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym = if result != nil: break t = t.sons[0] if t == nil: break - t = t.skipTypes(abstractInst) + t = t.skipTypes(skipPtrs) proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode = # returns a[].b as a node diff --git a/compiler/modules.nim b/compiler/modules.nim index 9120bd1b6..711fb6aa4 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -38,9 +38,6 @@ proc getModule*(fileIdx: int32): PSym = if fileIdx >= 0 and fileIdx < gCompiledModules.len: result = gCompiledModules[fileIdx] -template hash(x: PSym): untyped = - gMemCacheData[x.position].hash - proc hashChanged(fileIdx: int32): bool = internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len @@ -189,14 +186,11 @@ proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = return else: result.id = getID() - if sfMainModule in flags and gProjectIsStdin: - processModule(result, llStreamOpen(stdin), rd) - else: - processModule(result, nil, rd) + let validFile = processModule(result, if sfMainModule in flags and gProjectIsStdin: llStreamOpen(stdin) else: nil, rd) if optCaasEnabled in gGlobalOptions: gMemCacheData[fileIdx].compiledAt = gLastCmdTime gMemCacheData[fileIdx].needsRecompile = Recompiled - doHash fileIdx + if validFile: doHash fileIdx else: if checkDepMem(fileIdx) == Yes: result = compileModule(fileIdx, flags) @@ -220,12 +214,6 @@ proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} = addDep(s, fileIdx) doHash(fileIdx) -proc `==^`(a, b: string): bool = - try: - result = sameFile(a, b) - except OSError: - result = false - proc compileSystemModule* = if magicsys.systemModule == nil: systemFileIdx = fileInfoIdx(options.libpath/"system.nim") diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 4a9980066..fd0aafccb 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -65,7 +65,7 @@ type errPureTypeMismatch, errTypeMismatch, errButExpected, errButExpectedX, errAmbiguousCallXYZ, errWrongNumberOfArguments, errXCannotBePassedToProcVar, - errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProc, errImplOfXNotAllowed, + errXCannotBeInParamDecl, errPragmaOnlyInHeaderOfProcX, errImplOfXNotAllowed, errImplOfXexpected, errNoSymbolToBorrowFromFound, errDiscardValueX, errInvalidDiscard, errIllegalConvFromXtoY, errCannotBindXTwice, errInvalidOrderInArrayConstructor, @@ -274,7 +274,7 @@ const errWrongNumberOfArguments: "wrong number of arguments", errXCannotBePassedToProcVar: "\'$1\' cannot be passed to a procvar", errXCannotBeInParamDecl: "$1 cannot be declared in parameter declaration", - errPragmaOnlyInHeaderOfProc: "pragmas are only allowed in the header of a proc", + errPragmaOnlyInHeaderOfProcX: "pragmas are only allowed in the header of a proc; redefinition of $1", errImplOfXNotAllowed: "implementation of \'$1\' is not allowed", errImplOfXexpected: "implementation of \'$1\' expected", errNoSymbolToBorrowFromFound: "no symbol to borrow from found", @@ -521,7 +521,7 @@ const hintCodeBegin, hintCodeEnd, hintSource, hintStackTrace, hintGCStats}, - {low(TNoteKind)..high(TNoteKind)} - {hintStackTrace}, + {low(TNoteKind)..high(TNoteKind)} - {hintStackTrace, warnUninit}, {low(TNoteKind)..high(TNoteKind)}] const @@ -660,8 +660,6 @@ const WarningColor = fgYellow HintTitle = "Hint: " HintColor = fgGreen - InfoTitle = "Info: " - InfoColor = fgCyan proc getInfoContextLen*(): int = return msgContext.len proc setInfoContextLen*(L: int) = setLen(msgContext, L) diff --git a/compiler/nimblecmd.nim b/compiler/nimblecmd.nim index c6c2ab058..9c5c17287 100644 --- a/compiler/nimblecmd.nim +++ b/compiler/nimblecmd.nim @@ -63,18 +63,6 @@ proc addNimblePath(p: string, info: TLineInfo) = message(info, hintPath, p) lists.prependStr(options.lazyPaths, p) -proc addPathWithNimFiles(p: string, info: TLineInfo) = - proc hasNimFile(dir: string): bool = - for kind, path in walkDir(dir): - if kind == pcFile and path.endsWith(".nim"): - result = true - break - if hasNimFile(p): - addNimblePath(p, info) - else: - for kind, p2 in walkDir(p): - if hasNimFile(p2): addNimblePath(p2, info) - proc addPathRec(dir: string, info: TLineInfo) = var packages = newStringTable(modeStyleInsensitive) var pos = dir.len-1 diff --git a/compiler/nversion.nim b/compiler/nversion.nim index adeb0fb6d..d69e1e553 100644 --- a/compiler/nversion.nim +++ b/compiler/nversion.nim @@ -13,5 +13,5 @@ const MaxSetElements* = 1 shl 16 # (2^16) to support unicode character sets? VersionAsString* = system.NimVersion - RodFileVersion* = "1215" # modify this if the rod-format changes! + RodFileVersion* = "1221" # modify this if the rod-format changes! diff --git a/compiler/options.nim b/compiler/options.nim index a5154cb48..7cf707945 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -70,6 +70,12 @@ type # please make sure we have under 32 options optExcessiveStackTrace # fully qualified module filenames TGlobalOptions* = set[TGlobalOption] + +const + harmlessOptions* = {optForceFullMake, optNoLinking, optReportConceptFailures, + optRun, optUseColors, optStdout} + +type TCommands* = enum # Nim's commands # **keep binary compatible** cmdNone, cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, diff --git a/compiler/parser.nim b/compiler/parser.nim index 19ef0960a..40862eb63 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -340,26 +340,6 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = if not isKeyword(p.tok.tokType): getTok(p) result = ast.emptyNode -proc indexExpr(p: var TParser): PNode = - #| indexExpr = expr - result = parseExpr(p) - -proc indexExprList(p: var TParser, first: PNode, k: TNodeKind, - endToken: TTokType): PNode = - #| indexExprList = indexExpr ^+ comma - result = newNodeP(k, p) - addSon(result, first) - getTok(p) - optInd(p, result) - while p.tok.tokType notin {endToken, tkEof}: - var a = indexExpr(p) - addSon(result, a) - if p.tok.tokType != tkComma: break - getTok(p) - skipComment(p, a) - optPar(p) - eat(p, endToken) - proc colonOrEquals(p: var TParser, a: PNode): PNode = if p.tok.tokType == tkColon: result = newNodeP(nkExprColonExpr, p) diff --git a/compiler/passes.nim b/compiler/passes.nim index ceb3e2b8a..b7642e3e4 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -46,12 +46,6 @@ proc makePass*(open: TPassOpen = nil, result.close = close result.process = process - # This implements a memory preserving scheme: Top level statements are - # processed in a pipeline. The compiler never looks at a whole module - # any longer. However, this is simple to change, as new passes may perform - # whole program optimizations. For now, we avoid it to save a lot of memory. -proc processModule*(module: PSym, stream: PLLStream, rd: PRodReader) - # the semantic checker needs these: var gImportModule*: proc (m: PSym, fileIdx: int32): PSym {.nimcall.} @@ -160,7 +154,8 @@ proc processImplicits(implicits: seq[string], nodeKind: TNodeKind, importStmt.addSon str if not processTopLevelStmt(importStmt, a): break -proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) = +proc processModule*(module: PSym, stream: PLLStream, + rd: PRodReader): bool {.discardable.} = var p: TParsers a: TPassContextArray @@ -173,7 +168,7 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) = s = llStreamOpen(filename, fmRead) if s == nil: rawMessage(errCannotOpenFile, filename) - return + return false else: s = stream while true: @@ -211,4 +206,4 @@ proc processModule(module: PSym, stream: PLLStream, rd: PRodReader) = var n = loadInitSection(rd) for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a) closePassesCached(a) - + result = true diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index db5f8e727..f4109b26d 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -567,11 +567,13 @@ proc deprecatedStmt(c: PContext; pragma: PNode) = for n in pragma: if n.kind in {nkExprColonExpr, nkExprEqExpr}: let dest = qualifiedLookUp(c, n[1], {checkUndeclared}) + assert dest != nil let src = considerQuotedIdent(n[0]) let alias = newSym(skAlias, src, dest, n[0].info) incl(alias.flags, sfExported) if sfCompilerProc in dest.flags: markCompilerProc(alias) addInterfaceDecl(c, alias) + n.sons[1] = newSymNode(dest) else: localError(n.info, "key:value pair expected") @@ -749,7 +751,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: int, noVal(it) incl(sym.flags, sfThread) incl(sym.flags, sfProcvar) - if sym.typ != nil: incl(sym.typ.flags, tfThread) + if sym.typ != nil: + incl(sym.typ.flags, tfThread) + if sym.typ.callConv == ccClosure: sym.typ.callConv = ccDefault of wGcSafe: noVal(it) if sym.kind != skType: incl(sym.flags, sfThread) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 0e733d643..a116a8afe 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -146,12 +146,6 @@ proc put(g: var TSrcGen, kind: TTokType, s: string) = else: g.pendingWhitespace = s.len -proc putLong(g: var TSrcGen, kind: TTokType, s: string, lineLen: int) = - # use this for tokens over multiple lines. - addPendingNL(g) - addTok(g, kind, s) - g.lineLen = lineLen - proc toNimChar(c: char): string = case c of '\0': result = "\\0" @@ -264,9 +258,6 @@ proc pushCom(g: var TSrcGen, n: PNode) = proc popAllComs(g: var TSrcGen) = setLen(g.comStack, 0) -proc popCom(g: var TSrcGen) = - setLen(g.comStack, len(g.comStack) - 1) - const Space = " " @@ -492,7 +483,7 @@ proc fits(g: TSrcGen, x: int): bool = type TSubFlag = enum - rfLongMode, rfNoIndent, rfInConstExpr + rfLongMode, rfInConstExpr TSubFlags = set[TSubFlag] TContext = tuple[spacing: int, flags: TSubFlags] @@ -675,16 +666,6 @@ proc gfor(g: var TSrcGen, n: PNode) = gcoms(g) gstmts(g, n.sons[length - 1], c) -proc gmacro(g: var TSrcGen, n: PNode) = - var c: TContext - initContext(c) - gsub(g, n.sons[0]) - putWithSpace(g, tkColon, ":") - if longMode(n) or (lsub(n.sons[1]) + g.lineLen > MaxLineLen): - incl(c.flags, rfLongMode) - gcoms(g) - gsons(g, n, c, 1) - proc gcase(g: var TSrcGen, n: PNode) = var c: TContext initContext(c) diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 004b30b41..679e7ba15 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -145,13 +145,7 @@ type PRodReader* = ref TRodReader -var rodCompilerprocs*: TStrTable - -proc handleSymbolFile*(module: PSym): PRodReader -# global because this is needed by magicsys -proc loadInitSection*(r: PRodReader): PNode - -# implementation +var rodCompilerprocs*: TStrTable # global because this is needed by magicsys proc rawLoadStub(s: PSym) @@ -206,7 +200,7 @@ proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo, var id = decodeVInt(r.s, r.pos) result.typ = rrGetType(r, id, result.info) case result.kind - of nkCharLit..nkInt64Lit: + of nkCharLit..nkUInt64Lit: if r.s[r.pos] == '!': inc(r.pos) result.intVal = decodeVBiggestInt(r.s, r.pos) @@ -324,6 +318,29 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType = result.align = decodeVInt(r.s, r.pos).int16 else: result.align = 2 + + if r.s[r.pos] == '\14': + inc(r.pos) + result.lockLevel = decodeVInt(r.s, r.pos).TLockLevel + else: + result.lockLevel = UnspecifiedLockLevel + + if r.s[r.pos] == '\15': + inc(r.pos) + result.destructor = rrGetSym(r, decodeVInt(r.s, r.pos), info) + if r.s[r.pos] == '\16': + inc(r.pos) + result.deepCopy = rrGetSym(r, decodeVInt(r.s, r.pos), info) + if r.s[r.pos] == '\17': + inc(r.pos) + result.assignment = rrGetSym(r, decodeVInt(r.s, r.pos), info) + while r.s[r.pos] == '\18': + inc(r.pos) + let x = decodeVInt(r.s, r.pos) + doAssert r.s[r.pos] == '\19' + inc(r.pos) + let y = rrGetSym(r, decodeVInt(r.s, r.pos), info) + result.methods.safeAdd((x, y)) decodeLoc(r, result.loc, info) while r.s[r.pos] == '^': inc(r.pos) @@ -349,6 +366,22 @@ proc decodeLib(r: PRodReader, info: TLineInfo): PLib = inc(r.pos) result.path = decodeNode(r, info) +proc decodeInstantiations(r: PRodReader; info: TLineInfo; + s: var seq[PInstantiation]) = + while r.s[r.pos] == '\15': + inc(r.pos) + var ii: PInstantiation + new ii + ii.sym = rrGetSym(r, decodeVInt(r.s, r.pos), info) + ii.concreteTypes = @[] + while r.s[r.pos] == '\17': + inc(r.pos) + ii.concreteTypes.add rrGetType(r, decodeVInt(r.s, r.pos), info) + if r.s[r.pos] == '\20': + inc(r.pos) + ii.compilesId = decodeVInt(r.s, r.pos) + s.safeAdd ii + proc decodeSym(r: PRodReader, info: TLineInfo): PSym = var id: int @@ -423,6 +456,27 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym = if r.s[r.pos] == '#': inc(r.pos) result.constraint = decodeNode(r, unknownLineInfo()) + case result.kind + of skType, skGenericParam: + while r.s[r.pos] == '\14': + inc(r.pos) + result.typeInstCache.safeAdd rrGetType(r, decodeVInt(r.s, r.pos), result.info) + of routineKinds: + decodeInstantiations(r, result.info, result.procInstCache) + if r.s[r.pos] == '\16': + inc(r.pos) + result.gcUnsafetyReason = rrGetSym(r, decodeVInt(r.s, r.pos), result.info) + of skModule, skPackage: + decodeInstantiations(r, result.info, result.usedGenerics) + of skLet, skVar, skField, skForVar: + if r.s[r.pos] == '\18': + inc(r.pos) + result.guard = rrGetSym(r, decodeVInt(r.s, r.pos), result.info) + if r.s[r.pos] == '\19': + inc(r.pos) + result.bitsize = decodeVInt(r.s, r.pos).int16 + else: discard + if r.s[r.pos] == '(': if result.kind in routineKinds: result.ast = decodeNodeLazyBody(r, result.info, result) @@ -566,7 +620,8 @@ proc processRodFile(r: PRodReader, hash: SecureHash) = of "GOPTIONS": inc(r.pos) # skip ':' var dep = cast[TGlobalOptions](int32(decodeVInt(r.s, r.pos))) - if gGlobalOptions != dep: r.reason = rrOptions + if gGlobalOptions-harmlessOptions != dep-harmlessOptions: + r.reason = rrOptions of "CMD": inc(r.pos) # skip ':' var dep = cast[TCommands](int32(decodeVInt(r.s, r.pos))) @@ -580,14 +635,14 @@ proc processRodFile(r: PRodReader, hash: SecureHash) = if not condsyms.isDefined(getIdent(w)): r.reason = rrDefines #MessageOut('not defined, but should: ' + w); if r.s[r.pos] == ' ': inc(r.pos) - if (d != countDefinedSymbols()): r.reason = rrDefines + if d != countDefinedSymbols(): r.reason = rrDefines of "FILES": inc(r.pos, 2) # skip "(\10" inc(r.line) while r.s[r.pos] != ')': - let relativePath = decodeStr(r.s, r.pos) - let resolvedPath = relativePath.findModule(r.origFile) - let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath + let finalPath = decodeStr(r.s, r.pos) + #let resolvedPath = relativePath.findModule(r.origFile) + #let finalPath = if resolvedPath.len > 0: resolvedPath else: relativePath r.files.add(finalPath.fileInfoIdx) inc(r.pos) # skip #10 inc(r.line) @@ -683,8 +738,11 @@ proc newRodReader(modfilename: string, hash: SecureHash, if version != RodFileVersion: # since ROD files are only for caching, no backwards compatibility is # needed + #echo "expected version ", version, " ", RodFileVersion + result.memfile.close result = nil else: + result.memfile.close result = nil proc rrGetType(r: PRodReader, id: int, info: TLineInfo): PType = @@ -762,7 +820,7 @@ proc rrGetSym(r: PRodReader, id: int, info: TLineInfo): PSym = result = decodeSymSafePos(r, d, info) if result != nil and result.kind == skStub: rawLoadStub(result) -proc loadInitSection(r: PRodReader): PNode = +proc loadInitSection*(r: PRodReader): PNode = if r.initIdx == 0 or r.dataIdx == 0: internalError("loadInitSection") var oldPos = r.pos r.pos = r.initIdx @@ -818,9 +876,8 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = var hash = getHash(fileIdx) gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles result = rrNone - var r: PRodReader = nil var rodfile = toGeneratedFile(filename.withPackageName, RodExt) - r = newRodReader(rodfile, hash, fileIdx) + var r = newRodReader(rodfile, hash, fileIdx) if r == nil: result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist) else: @@ -847,7 +904,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = gMods[fileIdx].rd = r gMods[fileIdx].reason = result # now we know better -proc handleSymbolFile(module: PSym): PRodReader = +proc handleSymbolFile*(module: PSym): PRodReader = let fileIdx = module.fileIdx if optSymbolFiles notin gGlobalOptions: module.id = getID() @@ -923,7 +980,7 @@ proc writeNode(f: File; n: PNode) = f.write('^') f.write(n.typ.id) case n.kind - of nkCharLit..nkInt64Lit: + of nkCharLit..nkUInt64Lit: if n.intVal != 0: f.write('!') f.write(n.intVal) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index d48a9ba40..addbdade6 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -16,8 +16,6 @@ import condsyms, ropes, idents, securehash, rodread, passes, importer, idgen, rodutils -# implementation - type TRodWriter = object of TPassContext module: PSym @@ -39,13 +37,6 @@ type PRodWriter = ref TRodWriter -proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter -proc addModDep(w: PRodWriter, dep: string) -proc addInclDep(w: PRodWriter, dep: string) -proc addInterfaceSym(w: PRodWriter, s: PSym) -proc addStmt(w: PRodWriter, n: PNode) -proc writeRod(w: PRodWriter) - proc getDefines(): string = result = "" for d in definedSymbolNames(): @@ -83,19 +74,20 @@ proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = result.converters = "" result.methods = "" result.init = "" - result.origFile = module.info.toFilename + result.origFile = module.info.toFullPath result.data = newStringOfCap(12_000) -proc addModDep(w: PRodWriter, dep: string) = +proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) = if w.modDeps.len != 0: add(w.modDeps, ' ') - encodeVInt(fileIdx(w, dep), w.modDeps) + let resolved = dep.findModule(info.toFullPath) + encodeVInt(fileIdx(w, resolved), w.modDeps) const rodNL = "\x0A" -proc addInclDep(w: PRodWriter, dep: string) = - var resolved = dep.findModule(w.module.info.toFullPath) - encodeVInt(fileIdx(w, dep), w.inclDeps) +proc addInclDep(w: PRodWriter, dep: string; info: TLineInfo) = + let resolved = dep.findModule(info.toFullPath) + encodeVInt(fileIdx(w, resolved), w.inclDeps) add(w.inclDeps, " ") encodeStr($secureHashFile(resolved), w.inclDeps) add(w.inclDeps, rodNL) @@ -108,6 +100,10 @@ proc pushType(w: PRodWriter, t: PType) = proc pushSym(w: PRodWriter, s: PSym) = # check so that the stack does not grow too large: if iiTableGet(w.index.tab, s.id) == InvalidKey: + when false: + if s.kind == skMethod: + echo "encoding ", s.id, " ", s.name.s + writeStackTrace() w.sstack.add(s) proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, @@ -127,7 +123,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, result.add(',') encodeVInt(n.info.line, result) result.add(',') - encodeVInt(fileIdx(w, toFilename(n.info)), result) + encodeVInt(fileIdx(w, toFullPath(n.info)), result) elif fInfo.line != n.info.line: result.add('?') encodeVInt(n.info.col, result) @@ -147,7 +143,7 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode, encodeVInt(n.typ.id, result) pushType(w, n.typ) case n.kind - of nkCharLit..nkInt64Lit: + of nkCharLit..nkUInt64Lit: if n.intVal != 0: result.add('!') encodeVBiggestInt(n.intVal, result) @@ -229,6 +225,27 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) = if t.align != 2: add(result, '=') encodeVInt(t.align, result) + if t.lockLevel.ord != UnspecifiedLockLevel.ord: + add(result, '\14') + encodeVInt(t.lockLevel.int16, result) + if t.destructor != nil and t.destructor.id != 0: + add(result, '\15') + encodeVInt(t.destructor.id, result) + pushSym(w, t.destructor) + if t.deepCopy != nil: + add(result, '\16') + encodeVInt(t.deepcopy.id, result) + pushSym(w, t.deepcopy) + if t.assignment != nil: + add(result, '\17') + encodeVInt(t.assignment.id, result) + pushSym(w, t.assignment) + for i, s in items(t.methods): + add(result, '\18') + encodeVInt(i, result) + add(result, '\19') + encodeVInt(s.id, result) + pushSym(w, s) encodeLoc(w, t.loc, result) for i in countup(0, sonsLen(t) - 1): if t.sons[i] == nil: @@ -246,6 +263,19 @@ proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) = add(result, '|') encodeNode(w, info, lib.path, result) +proc encodeInstantiations(w: PRodWriter; s: seq[PInstantiation]; + result: var string) = + for t in s: + result.add('\15') + encodeVInt(t.sym.id, result) + pushSym(w, t.sym) + for tt in t.concreteTypes: + result.add('\17') + encodeVInt(tt.id, result) + pushType(w, tt) + result.add('\20') + encodeVInt(t.compilesId, result) + proc encodeSym(w: PRodWriter, s: PSym, result: var string) = if s == nil: # nil nodes have to be stored too: @@ -266,7 +296,7 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = result.add(',') if s.info.line != -1'i16: encodeVInt(s.info.line, result) result.add(',') - encodeVInt(fileIdx(w, toFilename(s.info)), result) + encodeVInt(fileIdx(w, toFullPath(s.info)), result) if s.owner != nil: result.add('*') encodeVInt(s.owner.id, result) @@ -291,6 +321,31 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = if s.constraint != nil: add(result, '#') encodeNode(w, unknownLineInfo(), s.constraint, result) + case s.kind + of skType, skGenericParam: + for t in s.typeInstCache: + result.add('\14') + encodeVInt(t.id, result) + pushType(w, t) + of routineKinds: + encodeInstantiations(w, s.procInstCache, result) + if s.gcUnsafetyReason != nil: + result.add('\16') + encodeVInt(s.gcUnsafetyReason.id, result) + pushSym(w, s.gcUnsafetyReason) + of skModule, skPackage: + encodeInstantiations(w, s.usedGenerics, result) + # we don't serialize: + #tab*: TStrTable # interface table for modules + of skLet, skVar, skField, skForVar: + if s.guard != nil: + result.add('\18') + encodeVInt(s.guard.id, result) + pushSym(w, s.guard) + if s.bitsize != 0: + result.add('\19') + encodeVInt(s.bitsize, result) + else: discard # lazy loading will soon reload the ast lazily, so the ast needs to be # the last entry of a symbol: if s.ast != nil: @@ -298,16 +353,6 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = # it is not necessary, but Nim's heavy compile-time evaluation features # make that unfeasible nowadays: encodeNode(w, s.info, s.ast, result) - when false: - var codeAst: PNode = nil - if not astNeeded(s): - codeAst = s.ast.sons[codePos] - # ugly hack to not store the AST: - s.ast.sons[codePos] = ast.emptyNode - encodeNode(w, s.info, s.ast, result) - if codeAst != nil: - # resore the AST: - s.ast.sons[codePos] = codeAst proc addToIndex(w: var TIndex, key, val: int) = if key - w.lastIdxKey == 1: @@ -562,13 +607,15 @@ proc process(c: PPassContext, n: PNode): PNode = # addInterfaceSym(w, a.sons[j].sym); # end of nkImportStmt: - for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleName(n.sons[i])) + for i in countup(0, sonsLen(n) - 1): + addModDep(w, getModuleName(n.sons[i]), n.info) addStmt(w, n) - of nkFromStmt: - addModDep(w, getModuleName(n.sons[0])) + of nkFromStmt, nkImportExceptStmt: + addModDep(w, getModuleName(n.sons[0]), n.info) addStmt(w, n) of nkIncludeStmt: - for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleName(n.sons[i])) + for i in countup(0, sonsLen(n) - 1): + addInclDep(w, getModuleName(n.sons[i]), n.info) of nkPragma: addStmt(w, n) else: diff --git a/compiler/ropes.nim b/compiler/ropes.nim index bfae7aaa4..d84b59f78 100644 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -310,15 +310,19 @@ proc equalsFile*(r: Rope, f: File): bool = buf: array[bufSize, char] bpos = buf.len blen = buf.len + btotal = 0 + rtotal = 0 for s in leaves(r): var spos = 0 let slen = s.len + rtotal += slen while spos < slen: if bpos == blen: # Read more data bpos = 0 blen = readBuffer(f, addr(buf[0]), buf.len) + btotal += blen if blen == 0: # no more data in file result = false return @@ -330,7 +334,8 @@ proc equalsFile*(r: Rope, f: File): bool = spos += n bpos += n - result = readBuffer(f, addr(buf[0]), 1) == 0 # check that we've read all + result = readBuffer(f, addr(buf[0]), 1) == 0 and + btotal == rtotal # check that we've read all proc equalsFile*(r: Rope, filename: string): bool = ## returns true if the contents of the file `f` equal `r`. If `f` does not diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index 8b2653bc9..f04cef0ee 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -43,6 +43,7 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = template cbos(name, body) {.dirty.} = result.registerCallback "stdlib.system." & astToStr(name), proc (a: VmArgs) = + errorMsg = nil try: body except OSError: @@ -150,7 +151,7 @@ proc runNimScript*(scriptName: string; freshDefines=true) = vm.globalCtx = setupVM(m, scriptName) compileSystemModule() - processModule(m, llStreamOpen(scriptName, fmRead), nil) + discard processModule(m, llStreamOpen(scriptName, fmRead), nil) # ensure we load 'system.nim' again for the real non-config stuff! resetAllModulesHard() diff --git a/compiler/sem.nim b/compiler/sem.nim index 7db4ae47e..7768833b3 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -39,7 +39,6 @@ proc semStmt(c: PContext, n: PNode): PNode proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) proc addParams(c: PContext, n: PNode, kind: TSymKind) proc maybeAddResult(c: PContext, s: PSym, n: PNode) -proc instGenericContainer(c: PContext, n: PNode, header: PType): PType proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode proc activate(c: PContext, n: PNode) proc semQuoteAst(c: PContext, n: PNode): PNode @@ -47,6 +46,10 @@ proc finishMethod(c: PContext, s: PSym) proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode +proc isArrayConstr(n: PNode): bool {.inline.} = + result = n.kind == nkBracket and + n.typ.skipTypes(abstractInst).kind == tyArray + template semIdeForTemplateOrGenericCheck(n, requiresCheck) = # we check quickly if the node is where the cursor is when defined(nimsuggest): @@ -187,7 +190,6 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode, # identifier with visibility proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode, allowed: TSymFlags): PSym -proc semStmtScope(c: PContext, n: PNode): PNode proc typeAllowedCheck(info: TLineInfo; typ: PType; kind: TSymKind) = let t = typeAllowed(typ, kind) @@ -252,7 +254,7 @@ proc fixupTypeAfterEval(c: PContext, evaluated, eOrig: PNode): PNode = result = arg # for 'tcnstseq' we support [] to become 'seq' if eOrig.typ.skipTypes(abstractInst).kind == tySequence and - arg.typ.skipTypes(abstractInst).kind == tyArrayConstr: + isArrayConstr(arg): arg.typ = eOrig.typ proc tryConstExpr(c: PContext, n: PNode): PNode = @@ -445,6 +447,8 @@ proc isImportSystemStmt(n: PNode): bool = else: discard proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = + if n.kind == nkDefer: + localError(n.info, "defer statement not supported at top level") if c.topStmts == 0 and not isImportSystemStmt(n): if sfSystemModule notin c.module.flags and n.kind notin {nkEmpty, nkCommentStmt}: diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 9702128ba..2e925e386 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -203,7 +203,8 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = defaultOp(c, t, body, x, y) of tyObject, tyDistinct: if not considerOverloadedOp(c, t, body, x, y): - if t.sons[0] != nil: liftBodyAux(c, t.sons[0], body, x, y) + if t.sons[0] != nil: + liftBodyAux(c, t.sons[0].skipTypes(skipPtrs), body, x, y) if t.kind == tyObject: liftBodyObj(c, t.n, body, x, y) of tyTuple: liftBodyTup(c, t, body, x, y) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 97209167d..3037a6ecc 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -34,6 +34,22 @@ proc sameMethodDispatcher(a, b: PSym): bool = proc determineType(c: PContext, s: PSym) +proc initCandidateSymbols(c: PContext, headSymbol: PNode, + initialBinding: PNode, + filter: TSymKinds, + best, alt: var TCandidate, + o: var TOverloadIter): seq[tuple[s: PSym, scope: int]] = + result = @[] + var symx = initOverloadIter(o, c, headSymbol) + while symx != nil: + if symx.kind in filter: + result.add((symx, o.lastOverloadScope)) + symx = nextOverloadIter(o, c, headSymbol) + if result.len > 0: + initCandidate(c, best, result[0].s, initialBinding, result[0].scope) + initCandidate(c, alt, result[0].s, initialBinding, result[0].scope) + best.state = csNoMatch + proc pickBestCandidate(c: PContext, headSymbol: PNode, n, orig: PNode, initialBinding: PNode, @@ -41,66 +57,62 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, best, alt: var TCandidate, errors: var CandidateErrors) = var o: TOverloadIter - # thanks to the lazy semchecking for operands, we need to iterate over the - # symbol table *before* any call to 'initCandidate' which might invoke - # semExpr which might modify the symbol table in cases like - # 'init(a, 1, (var b = new(Type2); b))'. - var symx = initOverloadIter(o, c, headSymbol) - let symScope = o.lastOverloadScope - - var syms: seq[tuple[a: PSym, b: int]] = @[] - while symx != nil: - if symx.kind in filter: - syms.add((symx, o.lastOverloadScope)) - symx = nextOverloadIter(o, c, headSymbol) - if syms.len == 0: - when false: - if skIterator notin filter: - # also try iterators, but these are 2nd class: - symx = initOverloadIter(o, c, headSymbol) - while symx != nil: - if symx.kind == skIterator: - syms.add((symx, 100)) - symx = nextOverloadIter(o, c, headSymbol) - if syms.len == 0: return + var sym = initOverloadIter(o, c, headSymbol) + var scope = o.lastOverloadScope + # Thanks to the lazy semchecking for operands, we need to check whether + # 'initCandidate' modifies the symbol table (via semExpr). + # This can occur in cases like 'init(a, 1, (var b = new(Type2); b))' + let counterInitial = c.currentScope.symbols.counter + var syms: seq[tuple[s: PSym, scope: int]] + var nextSymIndex = 0 + while sym != nil: + if sym.kind in filter: + # Initialise 'best' and 'alt' with the first available symbol + initCandidate(c, best, sym, initialBinding, scope) + initCandidate(c, alt, sym, initialBinding, scope) + best.state = csNoMatch + break else: - return - + sym = nextOverloadIter(o, c, headSymbol) + scope = o.lastOverloadScope var z: TCandidate - initCandidate(c, best, syms[0][0], initialBinding, symScope) - initCandidate(c, alt, syms[0][0], initialBinding, symScope) - best.state = csNoMatch - - for i in 0 .. <syms.len: - let sym = syms[i][0] + while sym != nil: + if sym.kind notin filter: + sym = nextOverloadIter(o, c, headSymbol) + scope = o.lastOverloadScope + continue determineType(c, sym) - initCandidate(c, z, sym, initialBinding, syms[i][1]) - - #if sym.name.s == "*" and (n.info ?? "temp5.nim") and n.info.line == 140: - # gDebug = true - matches(c, n, orig, z) - if errors != nil: - errors.safeAdd((sym, int z.mutabilityProblem)) - if z.errors != nil: - for err in z.errors: - errors.add(err) - if z.state == csMatch: - # little hack so that iterators are preferred over everything else: - if sym.kind == skIterator: inc(z.exactMatches, 200) - case best.state - of csEmpty, csNoMatch: best = z - of csMatch: - var cmp = cmpCandidates(best, z) - if cmp < 0: best = z # x is better than the best so far - elif cmp == 0: alt = z # x is as good as the best so far - else: discard - #if sym.name.s == "cmp" and (n.info ?? "rstgen.nim") and n.info.line == 516: - # echo "Matches ", n.info, " ", typeToString(sym.typ) - # debug sym - # writeMatches(z) - # for i in 1 .. <len(z.call): - # z.call[i].typ.debug - # quit 1 + initCandidate(c, z, sym, initialBinding, scope) + if c.currentScope.symbols.counter == counterInitial or syms != nil: + matches(c, n, orig, z) + if errors != nil: + errors.safeAdd((sym, int z.mutabilityProblem)) + if z.errors != nil: + for err in z.errors: + errors.add(err) + if z.state == csMatch: + # little hack so that iterators are preferred over everything else: + if sym.kind == skIterator: inc(z.exactMatches, 200) + case best.state + of csEmpty, csNoMatch: best = z + of csMatch: + var cmp = cmpCandidates(best, z) + if cmp < 0: best = z # x is better than the best so far + elif cmp == 0: alt = z # x is as good as the best so far + else: + # Symbol table has been modified. Restart and pre-calculate all syms + # before any further candidate init and compare. SLOW, but rare case. + syms = initCandidateSymbols(c, headSymbol, initialBinding, filter, best, alt, o) + if syms == nil: + sym = nextOverloadIter(o, c, headSymbol) + scope = o.lastOverloadScope + elif nextSymIndex < syms.len: + # rare case: retrieve the next pre-calculated symbol + sym = syms[nextSymIndex].s + scope = syms[nextSymIndex].scope + nextSymIndex += 1 + else: + break proc notFoundError*(c: PContext, n: PNode, errors: CandidateErrors) = # Gives a detailed error message; this is separated from semOverloadedCall, @@ -203,6 +215,7 @@ proc resolveOverloads(c: PContext, n, orig: PNode, if result.state != csMatch: n.sons.delete(1) orig.sons.delete(1) + excl n.flags, nfExprCall else: return if nfDotField in n.flags: @@ -462,3 +475,9 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym = var resolved = semOverloadedCall(c, call, call, {fn.kind}) if resolved != nil: result = resolved.sons[0].sym + if not compareTypes(result.typ.sons[0], fn.typ.sons[0], dcEqIgnoreDistinct): + result = nil + elif result.magic in {mArrPut, mArrGet}: + # cannot borrow these magics for now + result = nil + diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index fc31829ba..fbbaaf483 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -136,8 +136,10 @@ proc isCastable(dst, src: PType): bool = # tyProc, tySet, tyEnum, tyBool, tyChar} if skipTypes(dst, abstractInst-{tyOpenArray}).kind == tyOpenArray: return false - var dstSize, srcSize: BiggestInt + if skipTypes(src, abstractInst-{tyTypeDesc}).kind == tyTypeDesc: + return false + var dstSize, srcSize: BiggestInt dstSize = computeSize(dst) srcSize = computeSize(src) if dstSize < 0: @@ -409,13 +411,11 @@ proc changeType(n: PNode, newType: PType, check: bool) = n.typ = newType proc arrayConstrType(c: PContext, n: PNode): PType = - var typ = newTypeS(tyArrayConstr, c) + var typ = newTypeS(tyArray, c) rawAddSon(typ, nil) # index type if sonsLen(n) == 0: rawAddSon(typ, newTypeS(tyEmpty, c)) # needs an empty basetype! else: - var x = n.sons[0] - var lastIndex: BiggestInt = sonsLen(n) - 1 var t = skipTypes(n.sons[0].typ, {tyGenericInst, tyVar, tyOrdinal}) addSonSkipIntLit(typ, t) typ.sons[0] = makeRangeType(c, 0, sonsLen(n) - 1, n.info) @@ -423,7 +423,7 @@ proc arrayConstrType(c: PContext, n: PNode): PType = proc semArrayConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = result = newNodeI(nkBracket, n.info) - result.typ = newTypeS(tyArrayConstr, c) + result.typ = newTypeS(tyArray, c) rawAddSon(result.typ, nil) # index type if sonsLen(n) == 0: rawAddSon(result.typ, newTypeS(tyEmpty, c)) # needs an empty basetype! @@ -468,59 +468,11 @@ proc fixAbstractType(c: PContext, n: PNode) = if it.kind == nkHiddenSubConv and skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}: if skipTypes(it.sons[1].typ, abstractVar).kind in - {tyNil, tyArrayConstr, tyTuple, tySet}: + {tyNil, tyTuple, tySet} or it[1].isArrayConstr: var s = skipTypes(it.typ, abstractVar) if s.kind != tyExpr: changeType(it.sons[1], s, check=true) n.sons[i] = it.sons[1] - when false: - # XXX finally rewrite that crap! - for i in countup(1, sonsLen(n) - 1): - var it = n.sons[i] - case it.kind - of nkHiddenStdConv, nkHiddenSubConv: - if it.sons[1].kind == nkBracket: - it.sons[1].typ = arrayConstrType(c, it.sons[1]) - #it.sons[1] = semArrayConstr(c, it.sons[1]) - if skipTypes(it.typ, abstractVar).kind in {tyOpenArray, tyVarargs}: - #if n.sons[0].kind == nkSym and IdentEq(n.sons[0].sym.name, "[]="): - # debug(n) - - var s = skipTypes(it.sons[1].typ, abstractVar) - if s.kind == tyArrayConstr and s.sons[1].kind == tyEmpty: - s = copyType(s, getCurrOwner(), false) - skipTypes(s, abstractVar).sons[1] = elemType( - skipTypes(it.typ, abstractVar)) - it.sons[1].typ = s - elif s.kind == tySequence and s.sons[0].kind == tyEmpty: - s = copyType(s, getCurrOwner(), false) - skipTypes(s, abstractVar).sons[0] = elemType( - skipTypes(it.typ, abstractVar)) - it.sons[1].typ = s - - elif skipTypes(it.sons[1].typ, abstractVar).kind in - {tyNil, tyArrayConstr, tyTuple, tySet}: - var s = skipTypes(it.typ, abstractVar) - if s.kind != tyExpr: - changeType(it.sons[1], s, check=true) - n.sons[i] = it.sons[1] - of nkBracket: - # an implicitly constructed array (passed to an open array): - n.sons[i] = semArrayConstr(c, it, {}) - else: - discard - #if (it.typ == nil): - # InternalError(it.info, "fixAbstractType: " & renderTree(it)) - -proc skipObjConv(n: PNode): PNode = - case n.kind - of nkHiddenStdConv, nkHiddenSubConv, nkConv: - if skipTypes(n.sons[1].typ, abstractPtrs).kind in {tyTuple, tyObject}: - result = n.sons[1] - else: - result = n - of nkObjUpConv, nkObjDownConv: result = n.sons[0] - else: result = n proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult = result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr) @@ -701,9 +653,6 @@ proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, # error correction, prevents endless for loop elimination in transf. # See bug #2051: result.sons[0] = newSymNode(errorSym(c, n)) - if sfNoSideEffect notin callee.flags: - if {sfImportc, sfSideEffect} * callee.flags != {}: - incl(c.p.owner.flags, sfSideEffect) proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode @@ -804,9 +753,6 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = else: result = m.call instGenericConvertersSons(c, result, m) - # we assume that a procedure that calls something indirectly - # has side-effects: - if tfNoSideEffect notin t.flags: incl(c.p.owner.flags, sfSideEffect) elif t != nil and t.kind == tyTypeDesc: if n.len == 1: return semObjConstr(c, n, flags) return semConv(c, n) @@ -986,7 +932,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = tyTuple, tySet, tyUInt..tyUInt64: if s.magic == mNone: result = inlineConst(n, s) else: result = newSymNode(s, n.info) - of tyArrayConstr, tySequence: + of tyArray, tySequence: # Consider:: # const x = [] # proc p(a: openarray[int]) @@ -1040,9 +986,6 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = markUsed(n.info, s) styleCheckUse(n.info, s) - # if a proc accesses a global variable, it is not side effect free: - if sfGlobal in s.flags: - incl(c.p.owner.flags, sfSideEffect) result = newSymNode(s, n.info) # We cannot check for access to outer vars for example because it's still # not sure the symbol really ends up being used: @@ -1091,7 +1034,7 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = result = check return result if ty.sons[0] == nil: break - ty = skipTypes(ty.sons[0], {tyGenericInst}) + ty = skipTypes(ty.sons[0], skipPtrs) # old code, not sure if it's live code: markUsed(n.info, s) styleCheckUse(n.info, s) @@ -1169,7 +1112,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode = f = lookupInRecordAndBuildCheck(c, n, ty.n, i, check) if f != nil: break if ty.sons[0] == nil: break - ty = skipTypes(ty.sons[0], {tyGenericInst}) + ty = skipTypes(ty.sons[0], skipPtrs) if f != nil: if fieldVisible(c, f): # is the access to a public field or in the same module or in a friend? @@ -1421,8 +1364,9 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode = # a = b # both are vars, means: a[] = b[] # a = b # b no 'var T' means: a = addr(b) var le = a.typ - if skipTypes(le, {tyGenericInst}).kind != tyVar and - isAssignable(c, a) == arNone: + if (skipTypes(le, {tyGenericInst}).kind != tyVar and + isAssignable(c, a) == arNone) or + skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}: # Direct assignment to a discriminant is allowed! localError(a.info, errXCannotBeAssignedTo, renderTree(a, {renderNoComments})) @@ -2128,7 +2072,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = f = lookupInRecordAndBuildCheck(c, it, t.n, id, check) if f != nil: break if t.sons[0] == nil: break - t = skipTypes(t.sons[0], {tyGenericInst}) + t = skipTypes(t.sons[0], skipPtrs) if f != nil and fieldVisible(c, f): it.sons[0] = newSymNode(f) e = fitNode(c, f.typ, e) @@ -2145,7 +2089,7 @@ proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode = while true: checkInitialized(objType.n, ids, n.info) if objType.sons[0] == nil: break - objType = skipTypes(objType.sons[0], {tyGenericInst}) + objType = skipTypes(objType.sons[0], skipPtrs) proc semBlock(c: PContext, n: PNode): PNode = result = n diff --git a/compiler/semfields.nim b/compiler/semfields.nim index 2e6c6c3ea..9d8cea862 100644 --- a/compiler/semfields.nim +++ b/compiler/semfields.nim @@ -152,7 +152,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode = while t.kind == tyObject: semForObjectFields(fc, t.n, n, stmts) if t.sons[0] == nil: break - t = skipTypes(t.sons[0], abstractPtrs) + t = skipTypes(t.sons[0], skipPtrs) dec(c.p.nestedLoopCounter) # for TR macros this 'while true: ...; break' loop is pretty bad, so # we avoid it now if we can: diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 02f238ae6..42fa60781 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -458,32 +458,6 @@ proc getConstIfExpr(c: PSym, n: PNode): PNode = if result == nil: result = getConstExpr(c, it.sons[0]) else: internalError(it.info, "getConstIfExpr()") -proc partialAndExpr(c: PSym, n: PNode): PNode = - # partial evaluation - result = n - var a = getConstExpr(c, n.sons[1]) - var b = getConstExpr(c, n.sons[2]) - if a != nil: - if getInt(a) == 0: result = a - elif b != nil: result = b - else: result = n.sons[2] - elif b != nil: - if getInt(b) == 0: result = b - else: result = n.sons[1] - -proc partialOrExpr(c: PSym, n: PNode): PNode = - # partial evaluation - result = n - var a = getConstExpr(c, n.sons[1]) - var b = getConstExpr(c, n.sons[2]) - if a != nil: - if getInt(a) != 0: result = a - elif b != nil: result = b - else: result = n.sons[2] - elif b != nil: - if getInt(b) != 0: result = b - else: result = n.sons[1] - proc leValueConv(a, b: PNode): bool = result = false case a.kind @@ -540,7 +514,8 @@ proc foldConv*(n, a: PNode; check = false): PNode = else: result = a result.typ = n.typ - if check: rangeCheck(n, result.intVal) + if check and result.kind in {nkCharLit..nkUInt64Lit}: + rangeCheck(n, result.intVal) of tyFloat..tyFloat64: case skipTypes(a.typ, abstractRange).kind of tyInt..tyInt64, tyEnum, tyBool, tyChar: diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 460db4f7c..d7cad6a2f 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -47,7 +47,7 @@ proc rawHandleSelf(c: PContext; owner: PSym) = while t.kind == tyObject: addObjFieldsToLocalScope(c, t.n) if t.sons[0] == nil: break - t = t.sons[0].skipTypes(abstractPtrs) + t = t.sons[0].skipTypes(skipPtrs) proc pushProcCon*(c: PContext; owner: PSym) = rawPushProcCon(c, owner) @@ -97,22 +97,6 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation; if inst.compilesId == id and sameInstantiation(entry, inst[]): return inst.sym -proc removeDefaultParamValues(n: PNode) = - # we remove default params, because they cannot be instantiated properly - # and they are not needed anyway for instantiation (each param is already - # provided). - when false: - for i in countup(1, sonsLen(n)-1): - var a = n.sons[i] - if a.kind != nkIdentDefs: IllFormedAst(a) - var L = a.len - if a.sons[L-1].kind != nkEmpty and a.sons[L-2].kind != nkEmpty: - # ``param: typ = defaultVal``. - # We don't need defaultVal for semantic checking and it's wrong for - # ``cmp: proc (a, b: T): int = cmp``. Hm, for ``cmp = cmp`` that is - # not possible... XXX We don't solve this issue here. - a.sons[L-1] = ast.emptyNode - proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig: @@ -128,17 +112,6 @@ proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind) -proc addProcDecls(c: PContext, fn: PSym) = - # get the proc itself in scope (e.g. for recursion) - addDecl(c, fn) - - for i in 1 .. <fn.typ.n.len: - var param = fn.typ.n.sons[i].sym - param.owner = fn - addParamOrResult(c, param, fn.kind) - - maybeAddResult(c, fn, fn.ast) - proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) = if n.sons[bodyPos].kind != nkEmpty: inc c.inGenericInst @@ -172,9 +145,10 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) = popInfoContext() proc sideEffectsCheck(c: PContext, s: PSym) = - if {sfNoSideEffect, sfSideEffect} * s.flags == - {sfNoSideEffect, sfSideEffect}: - localError(s.info, errXhasSideEffects, s.name.s) + when false: + if {sfNoSideEffect, sfSideEffect} * s.flags == + {sfNoSideEffect, sfSideEffect}: + localError(s.info, errXhasSideEffects, s.name.s) proc instGenericContainer(c: PContext, info: TLineInfo, header: PType, allowMetaTypes = false): PType = @@ -187,9 +161,6 @@ proc instGenericContainer(c: PContext, info: TLineInfo, header: PType, cl.allowMetaTypes = allowMetaTypes result = replaceTypeVarsT(cl, header) -proc instGenericContainer(c: PContext, n: PNode, header: PType): PType = - result = instGenericContainer(c, n.info, header) - proc instantiateProcType(c: PContext, pt: TIdTable, prc: PSym, info: TLineInfo) = # XXX: Instantiates a generic proc signature, while at the same diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index cbe9bc176..806b00db6 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -151,13 +151,6 @@ proc isStrangeArray(t: PType): bool = let t = t.skipTypes(abstractInst) result = t.kind == tyArray and t.firstOrd != 0 -proc isNegative(n: PNode): bool = - let n = n.skipConv - if n.kind in {nkCharLit..nkUInt64Lit}: - result = n.intVal < 0 - elif n.kind in nkCallKinds and n.sons[0].kind == nkSym: - result = n.sons[0].sym.magic in {mUnaryMinusI, mUnaryMinusI64} - proc magicsAfterOverloadResolution(c: PContext, n: PNode, flags: TExprFlags): PNode = case n[0].sym.magic diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index b12ab5e96..df9b3f69c 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -59,7 +59,7 @@ type init: seq[int] # list of initialized variables guards: TModel # nested guards locked: seq[PNode] # locked locations - gcUnsafe, isRecursive, isToplevel: bool + gcUnsafe, isRecursive, isToplevel, hasSideEffect: bool maxLockLevel, currLockLevel: TLockLevel PEffects = var TEffects @@ -131,7 +131,7 @@ proc guardDotAccess(a: PEffects; n: PNode) = if field != nil: break ty = ty.sons[0] if ty == nil: break - ty = ty.skipTypes(abstractPtrs) + ty = ty.skipTypes(skipPtrs) if field == nil: localError(n.info, errGenerated, "invalid guard field: " & g.name.s) return @@ -192,6 +192,14 @@ proc markGcUnsafe(a: PEffects; reason: PNode) = a.owner.gcUnsafetyReason = newSym(skUnknown, getIdent("<unknown>"), a.owner, reason.info) +when true: + template markSideEffect(a: PEffects; reason: typed) = + a.hasSideEffect = true +else: + template markSideEffect(a: PEffects; reason: typed) = + a.hasSideEffect = true + markGcUnsafe(a, reason) + proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) = let u = s.gcUnsafetyReason if u != nil and not cycleCheck.containsOrIncl(u.id): @@ -226,12 +234,16 @@ proc useVar(a: PEffects, n: PNode) = message(n.info, warnUninit, s.name.s) # prevent superfluous warnings about the same variable: a.init.add s.id - if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet}: + if {sfGlobal, sfThread} * s.flags != {} and s.kind in {skVar, skLet} and + s.magic != mNimVm: if s.guard != nil: guardGlobal(a, n, s.guard) if {sfGlobal, sfThread} * s.flags == {sfGlobal} and (tfHasGCedMem in s.typ.flags or s.typ.isGCedMem): #if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(a, s) + else: + markSideEffect(a, s) + type TIntersection = seq[tuple[id, count: int]] # a simple count table @@ -268,10 +280,6 @@ proc createTag(n: PNode): PNode = result.typ = sysTypeFromName"TEffect" if not n.isNil: result.info = n.info -proc createAnyGlobal(n: PNode): PNode = - result = newSymNode(anyGlobal) - result.info = n.info - proc addEffect(a: PEffects, e: PNode, useLineInfo=true) = assert e.kind != nkRaiseStmt var aa = a.exc @@ -495,6 +503,8 @@ proc propagateEffects(tracked: PEffects, n: PNode, s: PSym) = if notGcSafe(s.typ) and sfImportc notin s.flags: if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(tracked, s) + if tfNoSideEffect notin s.typ.flags: + markSideEffect(tracked, s) mergeLockLevels(tracked, n, s.getLockLevel) proc notNilCheck(tracked: PEffects, n: PNode, paramType: PType) = @@ -550,12 +560,16 @@ proc trackOperand(tracked: PEffects, n: PNode, paramType: PType) = if notGcSafe(op) and not isOwnedProcVar(a, tracked.owner): if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(tracked, a) + elif tfNoSideEffect notin op.flags and not isOwnedProcVar(a, tracked.owner): + markSideEffect(tracked, a) else: mergeEffects(tracked, effectList.sons[exceptionEffects], n) mergeTags(tracked, effectList.sons[tagEffects], n) if notGcSafe(op): if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(tracked, a) + elif tfNoSideEffect notin op.flags: + markSideEffect(tracked, a) notNilCheck(tracked, n, paramType) proc breaksBlock(n: PNode): bool = @@ -684,6 +698,7 @@ proc track(tracked: PEffects, n: PNode) = if a.sym == tracked.owner: tracked.isRecursive = true # even for recursive calls we need to check the lock levels (!): mergeLockLevels(tracked, n, a.sym.getLockLevel) + if sfSideEffect in a.sym.flags: markSideEffect(tracked, a) else: mergeLockLevels(tracked, n, op.lockLevel) var effectList = op.n.sons[0] @@ -702,6 +717,10 @@ proc track(tracked: PEffects, n: PNode) = if not (a.kind == nkSym and a.sym == tracked.owner): if warnGcUnsafe in gNotes: warnAboutGcUnsafe(n) markGcUnsafe(tracked, a) + if tfNoSideEffect notin op.flags and not importedFromC(a): + # and it's not a recursive call: + if not (a.kind == nkSym and a.sym == tracked.owner): + markSideEffect(tracked, a) for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}: # may not look like an assignment, but it is: @@ -793,9 +812,6 @@ proc track(tracked: PEffects, n: PNode) = proc subtypeRelation(spec, real: PNode): bool = result = safeInheritanceDiff(real.excType, spec.typ) <= 0 -proc symbolPredicate(spec, real: PNode): bool = - result = real.sym.id == spec.sym.id - proc checkRaisesSpec(spec, real: PNode, msg: string, hints: bool; effectPredicate: proc (a, b: PNode): bool {.nimcall.}) = # check that any real exception is listed in 'spec'; mark those as used; @@ -912,8 +928,15 @@ proc trackProc*(s: PSym, body: PNode) = else: listGcUnsafety(s, onlyWarning=true) #localError(s.info, warnGcUnsafe2, s.name.s) + if sfNoSideEffect in s.flags and t.hasSideEffect: + when false: + listGcUnsafety(s, onlyWarning=false) + else: + localError(s.info, errXhasSideEffects, s.name.s) if not t.gcUnsafe: s.typ.flags.incl tfGcSafe + if not t.hasSideEffect and sfSideEffect notin s.flags: + s.typ.flags.incl tfNoSideEffect if s.typ.lockLevel == UnspecifiedLockLevel: s.typ.lockLevel = t.maxLockLevel elif t.maxLockLevel > s.typ.lockLevel: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 5f9989ea2..ebcff643f 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -155,7 +155,10 @@ proc discardCheck(c: PContext, result: PNode) = else: var n = result while n.kind in skipForDiscardable: n = n.lastSon - localError(n.info, errDiscardValueX, result.typ.typeToString) + if result.typ.kind == tyProc: + localError(n.info, "value of type '" & result.typ.typeToString & "' has to be discarded; for a function call use ()") + else: + localError(n.info, errDiscardValueX, result.typ.typeToString) proc semIf(c: PContext, n: PNode): PNode = result = n @@ -659,7 +662,7 @@ proc semRaise(c: PContext, n: PNode): PNode = if n.sons[0].kind != nkEmpty: n.sons[0] = semExprWithType(c, n.sons[0]) var typ = n.sons[0].typ - if typ.kind != tyRef or typ.sons[0].kind != tyObject: + if typ.kind != tyRef or typ.lastSon.kind != tyObject: localError(n.info, errExprCannotBeRaised) proc addGenericParamListToScope(c: PContext, n: PNode) = @@ -1141,11 +1144,6 @@ type TProcCompilationSteps = enum stepRegisterSymbol, stepDetermineType, - stepCompileBody - -proc isForwardDecl(s: PSym): bool = - internalAssert s.kind == skProc - result = s.ast[bodyPos].kind != nkEmpty proc semProcAux(c: PContext, n: PNode, kind: TSymKind, validPragmas: TSpecialWords, @@ -1183,8 +1181,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, s.ast = n #s.scope = c.currentScope - # if typeIsDetermined: assert phase == stepCompileBody - # else: assert phase == stepDetermineType # before compiling the proc body, set as current the scope # where the proc was declared let oldScope = c.currentScope @@ -1234,7 +1230,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, implicitPragmas(c, s, n, validPragmas) else: if n.sons[pragmasPos].kind != nkEmpty: - localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProc) + localError(n.sons[pragmasPos].info, errPragmaOnlyInHeaderOfProcX, + "'" & proto.name.s & "' from " & $proto.info) if sfForward notin proto.flags: wrongRedefinition(n.info, proto.name.s) excl(proto.flags, sfForward) @@ -1403,6 +1400,11 @@ proc semMacroDef(c: PContext, n: PNode): PNode = if namePos >= result.safeLen: return result var s = result.sons[namePos].sym var t = s.typ + var allUntyped = true + for i in 1 .. t.n.len-1: + let param = t.n.sons[i].sym + if param.typ.kind != tyExpr: allUntyped = false + if allUntyped: incl(s.flags, sfAllUntyped) if t.sons[0] == nil: localError(n.info, errXNeedsReturnType, "macro") if n.sons[bodyPos].kind == nkEmpty: localError(n.info, errImplOfXexpected, s.name.s) @@ -1566,8 +1568,3 @@ proc semStmtList(c: PContext, n: PNode, flags: TExprFlags): PNode = proc semStmt(c: PContext, n: PNode): PNode = # now: simply an alias: result = semExprNoType(c, n) - -proc semStmtScope(c: PContext, n: PNode): PNode = - openScope(c) - result = semStmt(c, n) - closeScope(c) diff --git a/compiler/semtempl.nim b/compiler/semtempl.nim index 20b5071ac..dfe3ded0d 100644 --- a/compiler/semtempl.nim +++ b/compiler/semtempl.nim @@ -287,35 +287,6 @@ proc semTemplBodySons(c: var TemplCtx, n: PNode): PNode = for i in 0.. < n.len: result.sons[i] = semTemplBody(c, n.sons[i]) -proc wrapInBind(c: var TemplCtx; n: PNode; opr: string): PNode = - let ident = getIdent(opr) - if ident.id in c.toInject: return n - - let s = searchInScopes(c.c, ident) - if s != nil: - var callee: PNode - if contains(c.toBind, s.id): - callee = symChoice(c.c, n, s, scClosed) - elif contains(c.toMixin, s.name.id): - callee = symChoice(c.c, n, s, scForceOpen) - elif s.owner == c.owner and sfGenSym in s.flags: - # template tmp[T](x: var seq[T]) = - # var yz: T - incl(s.flags, sfUsed) - callee = newSymNode(s, n.info) - styleCheckUse(n.info, s) - else: - callee = semTemplSymbol(c.c, n, s) - - let call = newNodeI(nkCall, n.info) - call.add(callee) - for i in 0 .. n.len-1: call.add(n[i]) - result = newNodeI(nkBind, n.info, 2) - result.sons[0] = n - result.sons[1] = call - else: - result = n - proc oprIsRoof(n: PNode): bool = const roof = "^" case n.kind diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9d00c06ca..5f47bca5c 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -110,6 +110,8 @@ proc semContainer(c: PContext, n: PNode, kind: TTypeKind, kindStr: string, result = newOrPrevType(kind, prev, c) if sonsLen(n) == 2: var base = semTypeNode(c, n.sons[1], nil) + if base.kind == tyVoid: + localError(n.info, errTIsNotAConcreteType, typeToString(base)) addSonSkipIntLit(result, base) else: localError(n.info, errXExpectsOneTypeParam, kindStr) @@ -659,11 +661,12 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType = if n.sonsLen == 0: return newConstraint(c, tyObject) var check = initIntSet() var pos = 0 - var base: PType = nil + var base, realBase: PType = nil # n.sons[0] contains the pragmas (if any). We process these later... checkSonsLen(n, 3) if n.sons[1].kind != nkEmpty: - base = skipTypesOrNil(semTypeNode(c, n.sons[1].sons[0], nil), skipPtrs) + realBase = semTypeNode(c, n.sons[1].sons[0], nil) + base = skipTypesOrNil(realBase, skipPtrs) if base.isNil: localError(n.info, errIllegalRecursionInTypeX, "object") else: @@ -674,9 +677,10 @@ proc semObjectNode(c: PContext, n: PNode, prev: PType): PType = if concreteBase.kind != tyError: localError(n.sons[1].info, errInheritanceOnlyWithNonFinalObjects) base = nil + realBase = nil if n.kind != nkObjectTy: internalError(n.info, "semObjectNode") result = newOrPrevType(tyObject, prev, c) - rawAddSon(result, base) + rawAddSon(result, realBase) if result.n.isNil: result.n = newNodeI(nkRecList, n.info) else: @@ -1137,14 +1141,12 @@ proc semProcTypeWithScope(c: PContext, n: PNode, checkSonsLen(n, 2) openScope(c) result = semProcTypeNode(c, n.sons[0], nil, prev, kind, isType=true) + # start with 'ccClosure', but of course pragmas can overwrite this: + result.callConv = ccClosure # dummy symbol for `pragma`: var s = newSymS(kind, newIdentNode(getIdent("dummy"), n.info), c) s.typ = result - if n.sons[1].kind == nkEmpty or n.sons[1].len == 0: - if result.callConv == ccDefault: - result.callConv = ccClosure - #Message(n.info, warnImplicitClosure, renderTree(n)) - else: + if n.sons[1].kind != nkEmpty and n.sons[1].len > 0: pragma(c, s, n.sons[1], procTypePragmas) when useEffectSystem: setEffectsForProcType(result, n.sons[1]) closeScope(c) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 12620d55d..b42d58474 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -86,7 +86,7 @@ type proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym -proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode): PNode +proc replaceTypeVarsN*(cl: var TReplTypeVars, n: PNode; start=0): PNode template checkMetaInvariants(cl: TReplTypeVars, t: PType) = when false: @@ -151,7 +151,7 @@ proc reResolveCallsWithTypedescParams(cl: var TReplTypeVars, n: PNode): PNode = return n -proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode = +proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0): PNode = if n == nil: return result = copyNode(n) if n.typ != nil: @@ -195,7 +195,9 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode): PNode = var length = sonsLen(n) if length > 0: newSons(result, length) - for i in countup(0, length - 1): + if start > 0: + result.sons[0] = n.sons[0] + for i in countup(start, length - 1): result.sons[i] = replaceTypeVarsN(cl, n.sons[i]) proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym = @@ -462,8 +464,8 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType = r = skipTypes(r2, {tyPtr, tyRef}) result.sons[i] = r propagateToOwner(result, r) - - result.n = replaceTypeVarsN(cl, result.n) + # bug #4677: Do not instantiate effect lists + result.n = replaceTypeVarsN(cl, result.n, ord(result.kind==tyProc)) case result.kind of tyArray: diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index ec429968c..df5a76a57 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -352,18 +352,28 @@ proc handleFloatRange(f, a: PType): TTypeRelation = else: result = isIntConv else: result = isNone -proc isObjectSubtype(a, f: PType): int = +proc isObjectSubtype(c: var TCandidate; a, f, fGenericOrigin: PType): int = var t = a assert t.kind == tyObject var depth = 0 + var last = a while t != nil and not sameObjectTypes(f, t): assert t.kind == tyObject t = t.sons[0] if t == nil: break - t = skipTypes(t, {tyGenericInst}) + last = t + t = skipTypes(t, skipPtrs) inc depth if t != nil: + if fGenericOrigin != nil and last.kind == tyGenericInst and + last.len-1 == fGenericOrigin.len: + for i in countup(1, sonsLen(fGenericOrigin) - 1): + let x = PType(idTableGet(c.bindings, fGenericOrigin.sons[i])) + if x == nil: + put(c, fGenericOrigin.sons[i], last.sons[i]) result = depth + else: + result = -1 type SkippedPtr = enum skippedNone, skippedRef, skippedPtr @@ -840,12 +850,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyOpenArray, tyVarargs: result = typeRel(c, base(f), base(a)) if result < isGeneric: result = isNone - of tyArrayConstr: - if (f.sons[0].kind != tyGenericParam) and (a.sons[1].kind == tyEmpty): - result = isSubtype # [] is allowed here - elif typeRel(c, base(f), a.sons[1]) >= isGeneric: - result = isSubtype - of tyArray: + of tyArray, tyArrayConstr: if (f.sons[0].kind != tyGenericParam) and (a.sons[1].kind == tyEmpty): result = isSubtype elif typeRel(c, base(f), a.sons[1]) >= isGeneric: @@ -896,7 +901,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = result = isEqual # elif tfHasMeta in f.flags: result = recordRel(c, f, a) else: - var depth = isObjectSubtype(a, f) + var depth = isObjectSubtype(c, a, f, nil) if depth > 0: inc(c.inheritancePenalty, depth) result = isSubtype @@ -1012,6 +1017,15 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = result = isGeneric else: let genericBody = f.sons[0] + var askip = skippedNone + var fskip = skippedNone + let aobj = x.skipToObject(askip) + let fobj = genericBody.lastSon.skipToObject(fskip) + if fobj != nil and aobj != nil and askip == fskip: + let depth = isObjectSubtype(c, aobj, fobj, f) + if depth >= 0: + c.inheritancePenalty += depth + return if depth == 0: isGeneric else: isSubtype result = typeRel(c, genericBody, x) if result != isNone: # see tests/generics/tgeneric3.nim for an example that triggers this @@ -1075,7 +1089,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = of tyBuiltInTypeClass: considerPreviousT: let targetKind = f.sons[0].kind - if targetKind == a.skipTypes({tyRange, tyGenericInst}).kind or + if targetKind == a.skipTypes({tyRange, tyGenericInst, tyBuiltInTypeClass}).kind or (targetKind in {tyProc, tyPointer} and a.kind == tyNil): put(c, f, a) return isGeneric @@ -1310,8 +1324,9 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType, var call = newNodeI(nkCall, arg.info) call.add(f.n.copyTree) call.add(arg.copyTree) - result = c.semOverloadedCall(c, call, call, routineKinds) + result = c.semExpr(c, call) if result != nil: + if result.typ == nil: return nil # resulting type must be consistent with the other arguments: var r = typeRel(m, f.sons[0], result.typ) if r < isGeneric: return nil @@ -1320,13 +1335,6 @@ proc localConvMatch(c: PContext, m: var TCandidate, f, a: PType, if r == isGeneric: result.typ = getInstantiatedType(c, arg, m, base(f)) m.baseTypeMatch = true - # bug #4545: allow the call to go through a 'var T': - let vt = result.sons[0].typ.sons[1] - if vt.kind == tyVar: - let x = result.sons[1] - let va = newNodeIT(nkHiddenAddr, x.info, vt) - va.add x - result.sons[1] = va proc incMatches(m: var TCandidate; r: TTypeRelation; convMatch = 1) = case r @@ -1559,8 +1567,13 @@ proc paramTypesMatch*(m: var TCandidate, f, a: PType, proc setSon(father: PNode, at: int, son: PNode) = - if sonsLen(father) <= at: setLen(father.sons, at + 1) + let oldLen = father.len + if oldLen <= at: + setLen(father.sons, at + 1) father.sons[at] = son + # insert potential 'void' parameters: + #for i in oldLen ..< at: + # father.sons[i] = newNodeIT(nkEmpty, son.info, getSysType(tyVoid)) # we are allowed to modify the calling node in the 'prepare*' procs: proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode = @@ -1591,17 +1604,17 @@ proc prepareNamedParam(a: PNode) = a.sons[0] = newIdentNode(considerQuotedIdent(a.sons[0]), info) proc arrayConstr(c: PContext, n: PNode): PType = - result = newTypeS(tyArrayConstr, c) + result = newTypeS(tyArray, c) rawAddSon(result, makeRangeType(c, 0, 0, n.info)) addSonSkipIntLit(result, skipTypes(n.typ, {tyGenericInst, tyVar, tyOrdinal})) proc arrayConstr(c: PContext, info: TLineInfo): PType = - result = newTypeS(tyArrayConstr, c) + result = newTypeS(tyArray, c) rawAddSon(result, makeRangeType(c, 0, -1, info)) rawAddSon(result, newTypeS(tyEmpty, c)) # needs an empty basetype! proc incrIndexType(t: PType) = - assert t.kind == tyArrayConstr + assert t.kind == tyArray inc t.sons[0].n.sons[1].intVal template isVarargsUntyped(x): untyped = diff --git a/compiler/suggest.nim b/compiler/suggest.nim index c127b968c..52f00550b 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -273,7 +273,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) = while true: suggestObject(c, t.n, outputs) if t.sons[0] == nil: break - t = skipTypes(t.sons[0], {tyGenericInst}) + t = skipTypes(t.sons[0], skipPtrs) suggestOperations(c, n, typ, outputs) elif typ.kind == tyTuple and typ.n != nil: suggestSymList(c, typ.n, outputs) @@ -413,17 +413,16 @@ proc safeSemExpr*(c: PContext, n: PNode): PNode = result = ast.emptyNode proc suggestExpr*(c: PContext, node: PNode) = - if nfIsCursor notin node.flags: - if gTrackPos.line < 0: return - var cp = inCheckpoint(node.info) - if cp == cpNone: return + if gTrackPos.line < 0: return + var cp = inCheckpoint(node.info) + if cp == cpNone: return var outputs = 0 # This keeps semExpr() from coming here recursively: if c.compilesContextId > 0: return inc(c.compilesContextId) if gIdeCmd == ideSug: - var n = if nfIsCursor in node.flags: node else: findClosestDot(node) + var n = findClosestDot(node) if n == nil: n = node if n.kind == nkDotExpr: var obj = safeSemExpr(c, n.sons[0]) @@ -436,7 +435,7 @@ proc suggestExpr*(c: PContext, node: PNode) = suggestEverything(c, n, outputs) elif gIdeCmd == ideCon: - var n = if nfIsCursor in node.flags: node else: findClosestCall(node) + var n = findClosestCall(node) if n == nil: n = node if n.kind in nkCallKinds: var a = copyNode(n) diff --git a/compiler/transf.nim b/compiler/transf.nim index fc400c524..5cd5e298b 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -594,13 +594,6 @@ proc transformFor(c: PTransf, n: PNode): PTransNode = popTransCon(c) # echo "transformed: ", stmtList.PNode.renderTree -proc getMagicOp(call: PNode): TMagic = - if call.sons[0].kind == nkSym and - call.sons[0].sym.kind in {skProc, skMethod, skConverter}: - result = call.sons[0].sym.magic - else: - result = mNone - proc transformCase(c: PTransf, n: PNode): PTransNode = # removes `elif` branches of a case stmt # adds ``else: nil`` if needed for the code generator diff --git a/compiler/trees.nim b/compiler/trees.nim index fdd88c348..a629b3834 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -12,32 +12,21 @@ import ast, astalgo, lexer, msgs, strutils, wordrecg -proc hasSon(father, son: PNode): bool = - for i in countup(0, sonsLen(father) - 1): - if father.sons[i] == son: - return true - result = false - -proc cyclicTreeAux(n, s: PNode): bool = - if n == nil: - return false - if hasSon(s, n): - return true - var m = sonsLen(s) - addSon(s, n) +proc cyclicTreeAux(n: PNode, visited: var seq[PNode]): bool = + if n == nil: return + for v in visited: + if v == n: return true if not (n.kind in {nkEmpty..nkNilLit}): - for i in countup(0, sonsLen(n) - 1): - if cyclicTreeAux(n.sons[i], s): - return true - result = false - delSon(s, m) + visited.add(n) + for nSon in n.sons: + if cyclicTreeAux(nSon, visited): return true + discard visited.pop() proc cyclicTree*(n: PNode): bool = - var s = newNodeI(nkEmpty, n.info) - result = cyclicTreeAux(n, s) + var visited: seq[PNode] = @[] + cyclicTreeAux(n, visited) proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool = - result = false if a == b: result = true elif (a != nil) and (b != nil) and (a.kind == b.kind): @@ -61,7 +50,6 @@ proc exprStructuralEquivalent*(a, b: PNode; strictSymEquality=false): bool = result = true proc sameTree*(a, b: PNode): bool = - result = false if a == b: result = true elif a != nil and b != nil and a.kind == b.kind: @@ -84,17 +72,6 @@ proc sameTree*(a, b: PNode): bool = if not sameTree(a.sons[i], b.sons[i]): return result = true -proc getProcSym*(call: PNode): PSym = - result = call.sons[0].sym - -proc getOpSym*(op: PNode): PSym = - if op.kind notin {nkCall, nkHiddenCallConv, nkCommand, nkCallStrLit}: - result = nil - else: - if sonsLen(op) <= 0: internalError(op.info, "getOpSym") - elif op.sons[0].kind == nkSym: result = op.sons[0].sym - else: result = nil - proc getMagic*(op: PNode): TMagic = case op.kind of nkCallKinds: @@ -104,9 +81,8 @@ proc getMagic*(op: PNode): TMagic = else: result = mNone proc isConstExpr*(n: PNode): bool = - result = (n.kind in - {nkCharLit..nkInt64Lit, nkStrLit..nkTripleStrLit, - nkFloatLit..nkFloat64Lit, nkNilLit}) or (nfAllConst in n.flags) + const atomKinds = {nkCharLit..nkNilLit} # Char, Int, UInt, Str, Float and Nil literals + n.kind in atomKinds or nfAllConst in n.flags proc isCaseObj*(n: PNode): bool = if n.kind == nkRecCase: return true @@ -131,25 +107,6 @@ proc isDeepConstExpr*(n: PNode): bool = result = true else: discard -proc flattenTreeAux(d, a: PNode, op: TMagic) = - if (getMagic(a) == op): # a is a "leaf", so add it: - for i in countup(1, sonsLen(a) - 1): # BUGFIX - flattenTreeAux(d, a.sons[i], op) - else: - addSon(d, copyTree(a)) - -proc flattenTree*(root: PNode, op: TMagic): PNode = - result = copyNode(root) - if getMagic(root) == op: - # BUGFIX: forget to copy prc - addSon(result, copyNode(root.sons[0])) - flattenTreeAux(result, root, op) - -proc swapOperands*(op: PNode) = - var tmp = op.sons[1] - op.sons[1] = op.sons[2] - op.sons[2] = tmp - proc isRange*(n: PNode): bool {.inline.} = if n.kind in nkCallKinds: if n[0].kind == nkIdent and n[0].ident.id == ord(wDotDot) or @@ -168,7 +125,6 @@ proc unnestStmts(n, result: PNode) = result.add(n) proc flattenStmts*(n: PNode): PNode = - ## flattens a nested statement list; used for pattern matching result = newNodeI(nkStmtList, n.info) unnestStmts(n, result) if result.len == 1: diff --git a/compiler/types.nim b/compiler/types.nim index 312e36ae5..3db0c4507 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -94,7 +94,8 @@ proc invalidGenericInst(f: PType): bool = proc isPureObject(typ: PType): bool = var t = typ - while t.kind == tyObject and t.sons[0] != nil: t = t.sons[0] + while t.kind == tyObject and t.sons[0] != nil: + t = t.sons[0].skipTypes(skipPtrs) result = t.sym != nil and sfPure in t.sym.flags proc getOrdValue(n: PNode): BiggestInt = @@ -145,10 +146,6 @@ proc elemType*(t: PType): PType = else: result = t.lastSon assert(result != nil) -proc skipGeneric(t: PType): PType = - result = t - while result.kind == tyGenericInst: result = lastSon(result) - proc isOrdinalType(t: PType): bool = assert(t != nil) const @@ -232,7 +229,8 @@ proc searchTypeForAux(t: PType, predicate: TTypePredicate, if result: return case t.kind of tyObject: - result = searchTypeForAux(t.sons[0], predicate, marker) + if t.sons[0] != nil: + result = searchTypeForAux(t.sons[0].skipTypes(skipPtrs), predicate, marker) if not result: result = searchTypeNodeForAux(t.n, predicate, marker) of tyGenericInst, tyDistinct: result = searchTypeForAux(lastSon(t), predicate, marker) @@ -269,7 +267,9 @@ proc analyseObjectWithTypeFieldAux(t: PType, if searchTypeNodeForAux(t.n, isObjectWithTypeFieldPredicate, marker): return frEmbedded for i in countup(0, sonsLen(t) - 1): - res = analyseObjectWithTypeFieldAux(t.sons[i], marker) + var x = t.sons[i] + if x != nil: x = x.skipTypes(skipPtrs) + res = analyseObjectWithTypeFieldAux(x, marker) if res == frEmbedded: return frEmbedded if res == frHeader: result = frHeader @@ -578,10 +578,6 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = result = typeToStr[t.kind] result.addTypeFlags(t) -proc resultType(t: PType): PType = - assert(t.kind == tyProc) - result = t.sons[0] # nil is allowed - proc base(t: PType): PType = result = t.sons[0] @@ -661,7 +657,14 @@ proc lengthOrd(t: PType): BiggestInt = case t.kind of tyInt64, tyInt32, tyInt: result = lastOrd(t) of tyDistinct, tyConst, tyMutable: result = lengthOrd(t.sons[0]) - else: result = lastOrd(t) - firstOrd(t) + 1 + else: + let last = lastOrd t + let first = firstOrd t + # XXX use a better overflow check here: + if last == high(BiggestInt) and first <= 0: + result = last + else: + result = lastOrd(t) - firstOrd(t) + 1 # -------------- type equality ----------------------------------------------- @@ -772,18 +775,6 @@ proc equalParams(a, b: PNode): TParamsEquality = result = paramsIncompatible # overloading by different # result types does not work -proc sameLiteral(x, y: PNode): bool = - if x.kind == y.kind: - case x.kind - of nkCharLit..nkInt64Lit: result = x.intVal == y.intVal - of nkFloatLit..nkFloat64Lit: result = x.floatVal == y.floatVal - of nkNilLit: result = true - else: assert(false) - -proc sameRanges(a, b: PNode): bool = - result = sameLiteral(a.sons[0], b.sons[0]) and - sameLiteral(a.sons[1], b.sons[1]) - proc sameTuple(a, b: PType, c: var TSameTypeClosure): bool = # two tuples are equivalent iff the names, types and positions are the same; # however, both types may not have any field names (t.n may be nil) which @@ -1070,10 +1061,10 @@ proc typeAllowedNode(marker: var IntSet, n: PNode, kind: TSymKind, of nkNone..nkNilLit: discard else: + if n.kind == nkRecCase and kind in {skProc, skConst}: + return n[0].typ for i in countup(0, sonsLen(n) - 1): let it = n.sons[i] - if it.kind == nkRecCase and kind in {skProc, skConst}: - return n.typ result = typeAllowedNode(marker, it, kind, flags) if result != nil: break @@ -1306,7 +1297,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = a = maxAlign of tyObject: if typ.sons[0] != nil: - result = computeSizeAux(typ.sons[0], a) + result = computeSizeAux(typ.sons[0].skipTypes(skipPtrs), a) if result < 0: return maxAlign = a elif isObjectWithTypeFieldPredicate(typ): diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index d050a86b2..438744b1c 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -19,20 +19,14 @@ proc renderPlainSymbolName*(n: PNode): string = ## for the HTML hyperlinks. result = "" case n.kind - of nkPostfix: - for i in 0 .. <n.len: - result = renderPlainSymbolName(n[<n.len]) - if result.len > 0: - return + of nkPostfix, nkAccQuoted: + result = renderPlainSymbolName(n[<n.len]) of nkIdent: - if n.ident.s != "*": - result = n.ident.s + result = n.ident.s of nkSym: result = n.sym.renderDefinitionName(noQuotes = true) of nkPragmaExpr: result = renderPlainSymbolName(n[0]) - of nkAccQuoted: - result = renderPlainSymbolName(n[<n.len]) else: internalError(n.info, "renderPlainSymbolName() with " & $n.kind) assert(not result.isNil) diff --git a/compiler/vm.nim b/compiler/vm.nim index 5accf628e..efcc55c59 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -171,6 +171,7 @@ proc copyValue(src: PNode): PNode = result.info = src.info result.typ = src.typ result.flags = src.flags * PersistentNodeFlags + result.comment = src.comment when defined(useNodeIds): if result.id == nodeIdToDebug: echo "COMES FROM ", src.id @@ -269,6 +270,7 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame): c.currentExceptionA = nil # execute the corresponding handler: while c.code[pc2].opcode == opcExcept: inc pc2 + discard f.safePoints.pop return (pc2, f) inc pc2 if c.code[pc2].opcode != opcExcept and nextExceptOrFinally >= 0: @@ -282,7 +284,8 @@ proc cleanUpOnException(c: PCtx; tos: PStackFrame): pc2 = nextExceptOrFinally if c.code[pc2].opcode == opcFinally: # execute the corresponding handler, but don't quit walking the stack: - return (pc2, f) + discard f.safePoints.pop + return (pc2+1, f) # not the right one: discard f.safePoints.pop @@ -964,7 +967,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = # we'll execute in the 'raise' handler let rbx = instr.regBx - wordExcess - 1 # -1 for the following 'inc pc' inc pc, rbx - assert c.code[pc+1].opcode in {opcExcept, opcFinally} + while c.code[pc+1].opcode == opcExcept: + let rbx = c.code[pc+1].regBx - wordExcess - 1 + inc pc, rbx + #assert c.code[pc+1].opcode in {opcExcept, opcFinally} + if c.code[pc+1].opcode != opcFinally: + # in an except handler there is no active safe point for the 'try': + tos.popSafePoint() of opcFinally: # just skip it; it's followed by the code we need to execute anyway tos.popSafePoint() @@ -1063,7 +1072,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcRepr: decodeB(rkNode) createStr regs[ra] - regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments}) + regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments, renderDocComments}) of opcQuit: if c.mode in {emRepl, emStaticExpr, emStaticStmt}: message(c.debug[pc], hintQuitCalled) diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index 7dbf6f801..b40ca1058 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -109,10 +109,6 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; mapTypeToBracketX(name, m, t, info, inst) template newNodeX(kind): untyped = newNodeIT(kind, if t.n.isNil: info else: t.n.info, t) - template newIdent(s): untyped = - var r = newNodeX(nkIdent) - r.add !s - r template newIdentDefs(n,t): untyped = var id = newNodeX(nkIdentDefs) id.add n # name diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 61ab65360..6bfc33f00 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -748,7 +748,7 @@ proc genConv(c: PCtx; n, arg: PNode; dest: var TDest; opc=opcConv) = let tmp = c.genx(arg) if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opc, dest, tmp) - c.gABx(n, opc, 0, genType(c, n.typ)) + c.gABx(n, opc, 0, genType(c, n.typ.skipTypes({tyStatic}))) c.gABx(n, opc, 0, genType(c, arg.typ.skipTypes({tyStatic}))) c.freeTemp(tmp) @@ -1127,14 +1127,6 @@ proc fitsRegister*(t: PType): bool = t.skipTypes(abstractInst-{tyTypeDesc}).kind in { tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar} -proc requiresCopy(n: PNode): bool = - if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes: - result = false - elif n.kind in ({nkCurly, nkBracket, nkPar, nkObjConstr}+nkCallKinds): - result = false - else: - result = true - proc unneededIndirection(n: PNode): bool = n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind == tyRef @@ -1215,8 +1207,6 @@ proc whichAsgnOpc(n: PNode): TOpcode = else: opcAsgnComplex -proc isRef(t: PType): bool = t.skipTypes(abstractRange-{tyTypeDesc}).kind == tyRef - proc whichAsgnOpc(n: PNode; opc: TOpcode): TOpcode = opc proc genAsgn(c: PCtx; dest: TDest; ri: PNode; requiresCopy: bool) = @@ -1268,9 +1258,6 @@ proc isTemp(c: PCtx; dest: TDest): bool = template needsAdditionalCopy(n): untyped = not c.isTemp(dest) and not fitsRegister(n.typ) -proc skipDeref(n: PNode): PNode = - result = if n.kind in {nkDerefExpr, nkHiddenDeref}: n.sons[0] else: n - proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode; dest, idx, value: TRegister) = # opcLdObj et al really means "load address". We sometimes have to create a @@ -1848,7 +1835,6 @@ proc genParams(c: PCtx; params: PNode) = # res.sym.position is already 0 c.prc.slots[0] = (inUse: true, kind: slotFixedVar) for i in 1.. <params.len: - let param = params.sons[i].sym c.prc.slots[i] = (inUse: true, kind: slotFixedLet) c.prc.maxSlots = max(params.len, 1) diff --git a/config/nim.cfg b/config/nim.cfg index 0cc014a93..8c8270f3e 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -66,7 +66,7 @@ path="$lib/pure" @end @if unix: - @if not bsd: + @if not bsd or haiku: # -fopenmp gcc.options.linker = "-ldl" gcc.cpp.options.linker = "-ldl" @@ -80,6 +80,14 @@ path="$lib/pure" # at least NetBSD has problems with thread local storage: tlsEmulation:on @end + @if haiku: + # -fopenmp + gcc.options.linker = "-lroot -lnetwork" + gcc.cpp.options.linker = "-lroot -lnetwork" + clang.options.linker = "-lroot -lnetwork" + clang.cpp.options.linker = "-lroot -lnetwork" + tcc.options.linker = "-lroot -lnetwork" + @end @end # Configuration for the Intel C/C++ compiler: @@ -122,6 +130,17 @@ clang.objc.options.linker = "-lobjc -lgnustep-base" clang.objc.options.linker = "-framework Foundation" @end +# Options for FreeBSD, OpenBSD, NetBSD linker to add locations for searching +# shared libraries. +@if freebsd or openbsd or netbsd: + gcc.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" + gcc.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" + llvm_gcc.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" + llvm_gcc.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" + clang.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" + clang.cpp.options.linker = "-Wl,-rpath=.:/usr/local/lib:/usr/pkg/lib:/usr/X11R6/lib" +@end + # Configuration for the VxWorks # This has been tested with VxWorks 6.9 only @if vxworks: diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index 112833e58..832b4ffd0 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -49,7 +49,7 @@ $seeSrc # See doc.item for available substitution variables. doc.item.toc = """ <li><a class="reference" href="#$itemSymOrID" - title="$header_plain">$name</a></li> + title="$header_plain">$name<span class="attachedType" style="visibility:hidden">$attype</span></a></li> """ # HTML rendered for doc.item's seeSrc variable. Note that this will render to @@ -60,8 +60,10 @@ doc.item.toc = """ # * $url: whatever you did pass through the --docSeeSrcUrl switch (which also # gets variables path/line replaced!) doc.item.seesrc = """ <a -href="${url}/${path}#L${line}" -class="link-seesrc" target="_blank">Source</a>""" +href="${url}/tree/${commit}/${path}#L${line}" +class="link-seesrc" target="_blank">Source</a> +<a href="${url}/edit/devel/${path}#L${line}" class="link-seesrc" target="_blank" >Edit</a> +""" doc.toc = """ <ul class="simple simple-toc" id="toc-list"> @@ -72,9 +74,38 @@ $content doc.body_toc = """ <div class="row"> <div class="three columns"> + <div> + Search: <input type="text" id="searchInput" + onkeyup="search()" /> + </div> + $tableofcontents + </div> + <div class="nine columns" id="content"> + <div id="tocRoot"></div> + <p class="module-desc">$moduledesc</p> + $content + </div> +</div> +""" + +doc.body_toc_group = """ +<div class="row"> + <div class="three columns"> + <div> + Search: <input type="text" id="searchInput" + onkeyup="search()" /> + </div> + <div> + Group by: + <select onchange="groupBy(this.value)"> + <option value="section">Section</option> + <option value="type">Type</option> + </select> + </div> $tableofcontents </div> <div class="nine columns" id="content"> + <div id="tocRoot"></div> <p class="module-desc">$moduledesc</p> $content </div> @@ -93,7 +124,7 @@ doc.listing_end = "</pre>" doc.file = """<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<!-- This file is generated by Nimrod. --> +<!-- This file is generated by Nim. --> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> @@ -1244,8 +1275,16 @@ dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator span.pragmaend { cursor: pointer; } + +div.search_results { + background-color: antiquewhite; + margin: 3em; + padding: 1em; + border: 1px solid #4d4d4d; +} </style> +<script type="text/javascript" src="../dochack.js"></script> <script type="text/javascript"> function togglepragma(d) { diff --git a/doc/advopt.txt b/doc/advopt.txt index 02e69c5b8..b8980fa9c 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -3,6 +3,7 @@ Advanced commands: //compileToCpp, cpp compile project to C++ code //compileToOC, objc compile project to Objective C code //js compile project to Javascript + //e run a Nimscript file //rst2html convert a reStructuredText file to HTML //rst2tex convert a reStructuredText file to TeX //jsondoc extract the documentation to a json file diff --git a/doc/lib.rst b/doc/lib.rst index 66928055a..828889968 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -582,4 +582,4 @@ Nim programming language. nimblepkglist.js or have javascript disabled in your browser.</b></div> <script type="text/javascript" src="nimblepkglist.js"></script> - <script type="text/javascript" src="http://irclogs.nim-lang.org/packages?callback=gotPackageList"></script> + <script type="text/javascript" src="http://irclogs.nim-lang.org/packages?callback=gotPackageList" async></script> diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt index 4d03023c3..7ffd5eb1c 100644 --- a/doc/manual/lexing.txt +++ b/doc/manual/lexing.txt @@ -28,7 +28,7 @@ The parser uses a stack of indentation levels: the stack consists of integers counting the spaces. The indentation information is queried at strategic places in the parser but ignored otherwise: The pseudo terminal ``IND{>}`` denotes an indentation that consists of more spaces than the entry at the top -of the stack; IND{=} an indentation that has the same number of spaces. ``DED`` +of the stack; ``IND{=}`` an indentation that has the same number of spaces. ``DED`` is another pseudo terminal that describes the *action* of popping a value from the stack, ``IND{>}`` then implies to push onto the stack. @@ -399,7 +399,7 @@ These keywords are also operators: `=`:tok:, `:`:tok:, `::`:tok: are not available as general operators; they are used for other notational purposes. -``*:`` is as a special case the two tokens `*`:tok: and `:`:tok: +``*:`` is as a special case treated as the two tokens `*`:tok: and `:`:tok: (to support ``var v*: T``). diff --git a/doc/manual/pragmas.txt b/doc/manual/pragmas.txt index 70fc4a914..a19f41a34 100644 --- a/doc/manual/pragmas.txt +++ b/doc/manual/pragmas.txt @@ -161,7 +161,7 @@ requires full qualification. asmNoStackFrame pragma ---------------------- -A proc can be marked with the ``AsmNoStackFrame`` pragma to tell the compiler +A proc can be marked with the ``asmNoStackFrame`` pragma to tell the compiler it should not generate a stack frame for the proc. There are also no exit statements like ``return result;`` generated and the generated C function is declared as ``__declspec(naked)`` or ``__attribute__((naked))`` (depending on diff --git a/doc/manual/procs.txt b/doc/manual/procs.txt index ea6866845..181b1b1e5 100644 --- a/doc/manual/procs.txt +++ b/doc/manual/procs.txt @@ -61,14 +61,14 @@ Calling a procedure can be done in many different ways: .. code-block:: nim proc callme(x, y: int, s: string = "", c: char, b: bool = false) = ... - # call with positional arguments # parameter bindings: - callme(0, 1, "abc", '\t', true) # (x=0, y=1, s="abc", c='\t', b=true) + # call with positional arguments # parameter bindings: + callme(0, 1, "abc", '\t', true) # (x=0, y=1, s="abc", c='\t', b=true) # call with named and positional arguments: - callme(y=1, x=0, "abd", '\t') # (x=0, y=1, s="abd", c='\t', b=false) + callme(y=1, x=0, "abd", '\t') # (x=0, y=1, s="abd", c='\t', b=false) # call with named arguments (order is not relevant): - callme(c='\t', y=1, x=0) # (x=0, y=1, s="", c='\t', b=false) + callme(c='\t', y=1, x=0) # (x=0, y=1, s="", c='\t', b=false) # call as a command statement: no () needed: - callme 0, 1, "abc", '\t' + callme 0, 1, "abc", '\t' # (x=0, y=1, s="abc", c='\t', b=false) A procedure may call itself recursively. diff --git a/doc/manual/stmts.txt b/doc/manual/stmts.txt index 318738063..fa1cac8e1 100644 --- a/doc/manual/stmts.txt +++ b/doc/manual/stmts.txt @@ -462,10 +462,10 @@ While statement Example: .. code-block:: nim - echo "Please tell me your password: \n" + echo "Please tell me your password:" var pw = readLine(stdin) while pw != "12345": - echo "Wrong password! Next try: \n" + echo "Wrong password! Next try:" pw = readLine(stdin) diff --git a/doc/manual/syntax.txt b/doc/manual/syntax.txt index 89f8ca707..f3ace9f56 100644 --- a/doc/manual/syntax.txt +++ b/doc/manual/syntax.txt @@ -62,7 +62,8 @@ Precedence level Operators First charact ================ =============================================== ================== =============== -Whether an operator is used a prefix operator is also affected by preceeding whitespace (this parsing change was introduced with version 0.13.0): +Whether an operator is used a prefix operator is also affected by preceding +whitespace (this parsing change was introduced with version 0.13.0): .. code-block:: nim echo $foo diff --git a/doc/manual/templates.txt b/doc/manual/templates.txt index be5c6fa18..f01a703cd 100644 --- a/doc/manual/templates.txt +++ b/doc/manual/templates.txt @@ -406,11 +406,11 @@ builtin can be used for that: macro debug(n: varargs[typed]): untyped = result = newNimNode(nnkStmtList, n) - for i in 0..n.len-1: + for x in n: # we can bind symbols in scope via 'bindSym': - add(result, newCall(bindSym"write", bindSym"stdout", toStrLit(n[i]))) + add(result, newCall(bindSym"write", bindSym"stdout", toStrLit(x))) add(result, newCall(bindSym"write", bindSym"stdout", newStrLitNode(": "))) - add(result, newCall(bindSym"writeLine", bindSym"stdout", n[i])) + add(result, newCall(bindSym"writeLine", bindSym"stdout", x)) var a: array [0..10, int] diff --git a/doc/manual/type_sections.txt b/doc/manual/type_sections.txt index 34bbe6bd5..af761c77e 100644 --- a/doc/manual/type_sections.txt +++ b/doc/manual/type_sections.txt @@ -13,7 +13,7 @@ Example: Sym = object # a symbol name: string # the symbol's name line: int # the line the symbol was declared in - code: Node # the symbol's abstract syntax tree + code: Node # the symbol's abstract syntax tree A type section begins with the ``type`` keyword. It contains multiple type definitions. A type definition binds a type to a name. Type definitions diff --git a/doc/manual/types.txt b/doc/manual/types.txt index 1e2dc857f..02426e0d9 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -227,8 +227,8 @@ floating pointer values at compile time; this means expressions like Boolean type ------------ The boolean type is named `bool`:idx: in Nim and can be one of the two -pre-defined values ``true`` and ``false``. Conditions in while, -if, elif, when statements need to be of type bool. +pre-defined values ``true`` and ``false``. Conditions in ``while``, +``if``, ``elif``, ``when``-statements need to be of type ``bool``. This condition holds:: @@ -390,7 +390,10 @@ i-th *unichar*. The iterator ``runes`` from the `unicode module cstring type ------------ -The ``cstring`` type represents a pointer to a zero-terminated char array + +The ``cstring`` type meaning `compatible string` is the native representation +of a string for the compilation backend. For the C backend the ``cstring`` type +represents a pointer to a zero-terminated char array compatible to the type ``char*`` in Ansi C. Its primary purpose lies in easy interfacing with C. The index operation ``s[i]`` means the i-th *char* of ``s``; however no bounds checking for ``cstring`` is performed making the @@ -421,7 +424,6 @@ string from a cstring: var cstr: cstring = str var newstr: string = $cstr - Structured types ---------------- A variable of a structured type can hold multiple values at the same @@ -607,7 +609,7 @@ is similar to the ``instanceof`` operator in Java. age: int # no * means that the field is hidden Student = ref object of Person # a student is a person - id: int # with an id field + id: int # with an id field var student: Student @@ -1260,4 +1262,4 @@ Is the same as: However later versions of the language might change this to mean "infer the parameters' types from the body". Then the above ``foo`` would be rejected as -the parameters' types can not be infered from an empty ``discard`` statement. +the parameters' types can not be inferred from an empty ``discard`` statement. diff --git a/doc/nimsuggest.rst b/doc/nimsuggest.rst index 2b52196b9..4da39b5d0 100644 --- a/doc/nimsuggest.rst +++ b/doc/nimsuggest.rst @@ -80,7 +80,7 @@ a location. A query location consists of: ``col`` An integer with the column you are going to query. For the - compiler columns start at **1**. + compiler columns start at **0**. Definitions @@ -160,7 +160,7 @@ tab characters (``\t``). The values of each column are: 6. Line where the symbol is located in the file. Lines start to count at **1**. 7. Column where the symbol is located in the file. Columns start - to count at **1**. + to count at **0**. 8. Docstring for the symbol if available or the empty string. To differentiate the docstring from end of answer, the docstring is always provided enclosed in double quotes, and diff --git a/doc/sets_fragment.txt b/doc/sets_fragment.txt index a6fe7555d..435807e1a 100644 --- a/doc/sets_fragment.txt +++ b/doc/sets_fragment.txt @@ -1,5 +1,5 @@ -The set type models the mathematical notion of a set. The set's -basetype can only be an ordinal type of a certain size, namely: +The set type models the mathematical notion of a set. The set's basetype can +only be an ordinal type of a certain size, namely: * ``int8``-``int16`` * ``uint8``/``byte``-``uint16`` * ``char`` diff --git a/doc/tut2.rst b/doc/tut2.rst index 3f94325ff..845feb6d1 100644 --- a/doc/tut2.rst +++ b/doc/tut2.rst @@ -674,17 +674,18 @@ variable number of arguments: macro debug(n: varargs[expr]): stmt = # `n` is a Nim AST that contains a list of expressions; - # this macro returns a list of statements: + # this macro returns a list of statements (n is passed for proper line + # information): result = newNimNode(nnkStmtList, n) # iterate over any argument that is passed to this macro: - for i in 0..n.len-1: + for x in n: # add a call to the statement list that writes the expression; # `toStrLit` converts an AST to its string representation: - result.add(newCall("write", newIdentNode("stdout"), toStrLit(n[i]))) + result.add(newCall("write", newIdentNode("stdout"), toStrLit(x))) # add a call to the statement list that writes ": " result.add(newCall("write", newIdentNode("stdout"), newStrLitNode(": "))) # add a call to the statement list that writes the expressions value: - result.add(newCall("writeLine", newIdentNode("stdout"), n[i])) + result.add(newCall("writeLine", newIdentNode("stdout"), x)) var a: array[0..10, int] diff --git a/finish.nim b/finish.nim new file mode 100644 index 000000000..34580e171 --- /dev/null +++ b/finish.nim @@ -0,0 +1,158 @@ + +# -------------- post unzip steps --------------------------------------------- + +import strutils, os, osproc + +when defined(windows): + import registry + + proc askBool(m: string): bool = + stdout.write m + while true: + let answer = stdin.readLine().normalize + case answer + of "y", "yes": + return true + of "n", "no": + return false + else: + echo "Please type 'y' or 'n'" + + proc askNumber(m: string; a, b: int): int = + stdout.write m + stdout.write " [" & $a & ".." & $b & "] " + while true: + let answer = stdin.readLine() + try: + result = parseInt answer + if result < a or result > b: + raise newException(ValueError, "number out of range") + break + except ValueError: + echo "Please type in a number between ", a, " and ", b + + proc patchConfig(mingw: string) = + const + cfgFile = "config/nim.cfg" + lookFor = """#gcc.path = r"$nim\dist\mingw\bin"""" + replacePattern = """gcc.path = r"$1"""" + try: + let cfg = readFile(cfgFile) + let newCfg = cfg.replace(lookFor, replacePattern % mingw) + if newCfg == cfg: + echo "Could not patch 'config/nim.cfg' [Error]" + echo "Reason: patch substring not found:" + echo lookFor + else: + writeFile(cfgFile, newCfg) + except IOError: + echo "Could not access 'config/nim.cfg' [Error]" + + proc addToPathEnv(e: string) = + let p = getUnicodeValue(r"Environment", "Path", HKEY_CURRENT_USER) + let x = if e.contains(Whitespace): "\"" & e & "\"" else: e + setUnicodeValue(r"Environment", "Path", p & ";" & x, HKEY_CURRENT_USER) + + proc createShortcut(src, dest: string; icon = "") = + var cmd = "bin\\makelink.exe \"" & src & "\" \"\" \"" & dest & + ".lnk\" \"\" 1 \"" & splitFile(src).dir & "\"" + if icon.len != 0: + cmd.add " \"" & icon & "\" 0" + discard execShellCmd(cmd) + + proc createStartMenuEntry() = + let appdata = getEnv("APPDATA") + if appdata.len == 0: return + let dest = appdata & r"\Microsoft\Windows\Start Menu\Programs\Nim-" & + NimVersion + if dirExists(dest): return + if askBool("Would like to add Nim-" & NimVersion & + " to your start menu? (y/n) "): + createDir(dest) + createShortcut(getCurrentDir() / "start.bat", dest / "Nim", + getCurrentDir() / r"icons\nim.ico") + if fileExists("doc/overview.html"): + createShortcut(getCurrentDir() / "doc" / "overview.html", + dest / "Overview") + if dirExists(r"dist\aporia-0.4.0"): + createShortcut(getCurrentDir() / r"dist\aporia-0.4.0\bin\aporia.exe", + dest / "Aporia") + + proc checkGccArch(mingw: string): bool = + let gccExe = mingw / r"gcc.exe" + if fileExists(gccExe): + try: + let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut, + poUsePath}) + when hostCPU == "i386": + result = arch.startsWith("i686-") + elif hostCPU == "amd64": + result = arch.startsWith("x86_64-") + else: + {.error: "Unknown CPU for Windows.".} + except OSError, IOError: + result = false + + proc tryDirs(dirs: varargs[string]): string = + let bits = $(sizeof(pointer)*8) + for d in dirs: + if dirExists d: + let x = expandFilename(d / "bin") + if checkGccArch(x): return x + elif dirExists(d & bits): + let x = expandFilename((d & bits) / "bin") + if checkGccArch(x): return x + +proc main() = + when defined(windows): + let desiredPath = expandFilename(getCurrentDir() / "bin") + let p = getUnicodeValue(r"Environment", "Path", + HKEY_CURRENT_USER) + var alreadyInPath = false + var mingWchoices: seq[string] = @[] + for x in p.split(';'): + let y = expandFilename(if x[0] == '"' and x[^1] == '"': + substr(x, 1, x.len-2) else: x) + if y == desiredPath: alreadyInPath = true + if y.toLowerAscii.contains("mingw"): + if dirExists(y) and checkGccArch(y): + mingWchoices.add y + + if alreadyInPath: + echo "bin/nim.exe is already in your PATH [Skipping]" + else: + if askBool("nim.exe is not in your PATH environment variable.\n" & + " Should it be added permanently? (y/n) "): + addToPathEnv(desiredPath) + if mingWchoices.len == 0: + # No mingw in path, so try a few locations: + let alternative = tryDirs("dist/mingw", "../mingw", r"C:\mingw") + if alternative.len == 0: + echo "No MingW found in PATH and no candidate found " & + " in the standard locations [Error]" + else: + if askBool("Found a MingW directory that is not in your PATH.\n" & + alternative & + "\nShould it be added to your PATH permanently? (y/n) "): + addToPathEnv(alternative) + elif askBool("Do you want to patch Nim's config to use this? (y/n) "): + patchConfig(alternative) + elif mingWchoices.len == 1: + if askBool("MingW installation found at " & mingWchoices[0] & "\n" & + "Do you want to patch Nim's config to use this?\n" & + "(Not required since it's in your PATH!) (y/n) "): + patchConfig(mingWchoices[0]) + else: + echo "Multiple MingW installations found: " + for i in 0..high(mingWchoices): + echo "[", i, "] ", mingWchoices[i] + if askBool("Do you want to patch Nim's config to use one of these? (y/n) "): + let idx = askNumber("Which one do you want to use for Nim? ", + 1, len(mingWchoices)) + patchConfig(mingWchoices[idx-1]) + createStartMenuEntry() + else: + echo("Add ", getCurrentDir(), "/bin to your PATH...") + +when isMainModule: + main() diff --git a/install.txt b/install.txt index 67be2221b..833fbf0fb 100644 --- a/install.txt +++ b/install.txt @@ -30,6 +30,15 @@ There are also ``install.sh`` and ``deinstall.sh`` scripts for distributing the files over the UNIX hierarchy. However, updating your Nim installation is more cumbersome then. +To complete the installation you should also build Nim's tools like +``nimsuggest``, ``nimble`` or ``nimgrep`` via:: + + nim e install_tools.nims + +Note that these tools should also end up in your ``PATH`` so adding +``$your_install_dir/bin/nim`` to your ``PATH`` is preferred over the symlink +solution. + Installation on the Macintosh ----------------------------- diff --git a/install_nimble.nims b/install_nimble.nims index 5d028726b..05024c046 100644 --- a/install_nimble.nims +++ b/install_nimble.nims @@ -16,4 +16,8 @@ mkDir "bin/nimblepkg" for file in listFiles("nimble" & $id & "/src/nimblepkg/"): cpFile file, "bin/nimblepkg/" & file.extractFilename -mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe +try: + mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe +except OSError: + cpFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe + diff --git a/install_tools.nims b/install_tools.nims new file mode 100644 index 000000000..e211ff491 --- /dev/null +++ b/install_tools.nims @@ -0,0 +1,18 @@ + +import ospaths + +mode = ScriptMode.Verbose + +if not dirExists"dist/nimble": + echo "[Error] This script only works for the tarball." +else: + let nimbleExe = "./bin/nimble".toExe + selfExec "c --noNimblePath -p:compiler -o:" & nimbleExe & + " dist/nimble/src/nimble.nim" + + let nimsugExe = "./bin/nimsuggest".toExe + selfExec "c --noNimblePath -p:compiler -o:" & nimsugExe & + " dist/nimsuggest/nimsuggest.nim" + + let nimgrepExe = "./bin/nimgrep".toExe + selfExec "c -o:./bin/nimgrep tools/nimgrep.nim" diff --git a/koch.nim b/koch.nim index 04f6a4e4e..25c02937f 100644 --- a/koch.nim +++ b/koch.nim @@ -20,11 +20,6 @@ import const VersionAsString = system.NimVersion #"0.10.2" -when defined(withUpdate): - import httpclient -when defined(haveZipLib): - import zipfiles - const HelpText = """ +-----------------------------------------------------------------+ @@ -40,7 +35,7 @@ Options: --help, -h shows this help and quits Possible Commands: boot [options] bootstraps with given command line options - install [bindir] installs to given directory; Unix only! + distrohelper [bindir] helper for distro packagers geninstall generate ./install.sh; Unix only! testinstall test tar.xz package; Unix only! Only for devs! clean cleans Nim project; removes generated files @@ -52,8 +47,6 @@ Possible Commands: xz builds the installation XZ package nsis [options] builds the NSIS Setup installer (for Windows) tests [options] run the testsuite - update updates nim to the latest version from github - (compile koch with -d:withUpdate to enable) temp options creates a temporary compiler for testing winrelease creates a release (for coredevs only) Boot options: @@ -69,7 +62,10 @@ Web options: build the official docs, use UA-48159761-1 """ -proc exe(f: string): string = return addFileExt(f, ExeExt) +proc exe(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') proc findNim(): string = var nim = "nim".exe @@ -123,6 +119,8 @@ proc testUnixInstall() = execCleanPath("./koch boot -d:release", destDir / "bin") # check the docs build: execCleanPath("./koch web", destDir / "bin") + # check nimble builds: + execCleanPath("./bin/nim e install_tools.nims") # check the tests work: execCleanPath("./koch tests", destDir / "bin") else: @@ -143,13 +141,13 @@ proc copyExe(source, dest: string) = inclFilePermissions(dest, {fpUserExec}) const - compileNimInst = "-d:useLibzipSrc tools/niminst/niminst" + compileNimInst = "tools/niminst/niminst" proc csource(args: string) = exec("$4 cc $1 -r $3 --var:version=$2 --var:mingw=none csource --main:compiler/nim.nim compiler/installer.ini $1" % [args, VersionAsString, compileNimInst, findNim()]) -proc bundleNimble() = +proc bundleNimbleSrc() = if dirExists("dist/nimble/.git"): exec("git --git-dir dist/nimble/.git pull") else: @@ -157,20 +155,38 @@ proc bundleNimble() = let tags = execProcess("git --git-dir dist/nimble/.git tag -l v*").splitLines let tag = tags[^1] exec("git --git-dir dist/nimble/.git checkout " & tag) + +proc bundleNimbleExe() = + bundleNimbleSrc() # now compile Nimble and copy it to $nim/bin for the installer.ini # to pick it up: exec(findNim() & " c dist/nimble/src/nimble.nim") copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe) +proc bundleNimsuggest(buildExe: bool) = + if dirExists("dist/nimsuggest/.git"): + exec("git --git-dir dist/nimsuggest/.git pull") + else: + exec("git clone https://github.com/nim-lang/nimsuggest.git dist/nimsuggest") + if buildExe: + exec(findNim() & " c --noNimblePath -p:compiler dist/nimsuggest/nimsuggest.nim") + copyExe("dist/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) + +proc bundleFinishExe() = + exec(findNim() & " c finish.nim") + proc zip(args: string) = - bundleNimble() + bundleNimbleSrc() + bundleNimsuggest(false) + bundleFinishExe() exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) proc xz(args: string) = - bundleNimble() + bundleNimbleSrc() + bundleNimsuggest(false) exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % [VersionAsString, compileNimInst, findNim()]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % @@ -181,7 +197,9 @@ proc buildTool(toolname, args: string) = copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe) proc nsis(args: string) = - bundleNimble() + bundleNimbleExe() + bundleNimsuggest(true) + bundleFinishExe() # make sure we have generated the niminst executables: buildTool("tools/niminst/niminst", args) #buildTool("tools/nimgrep", args) @@ -195,11 +213,8 @@ proc geninstall(args="") = exec("$# cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini $#" % [findNim(), compileNimInst, VersionAsString, args]) -proc install(args: string) = - geninstall() - exec("sh ./install.sh $#" % args) - proc web(args: string) = + exec("$# js tools/dochack/dochack.nim" % findNim()) exec("$# cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % [findNim(), args, VersionAsString]) @@ -244,11 +259,13 @@ proc boot(args: string) = var finalDest = "bin" / "nim".exe # default to use the 'c' command: let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: "" + let smartNimcache = if "release" in args: "rnimcache" else: "dnimcache" copyExe(findStartNim(), 0.thVersion) for i in 0..2: echo "iteration: ", i+1 - exec i.thVersion & " $# $# compiler" / "nim.nim" % [bootOptions, args] + exec i.thVersion & " $# $# --nimcache:$# compiler" / "nim.nim" % [bootOptions, args, + smartNimcache] if sameFileContent(output, i.thVersion): copyExe(output, finalDest) echo "executables are equal: SUCCESS!" @@ -273,7 +290,7 @@ proc cleanAux(dir: string) = for kind, path in walkDir(dir): case kind of pcFile: - var (dir, name, ext) = splitFile(path) + var (_, name, ext) = splitFile(path) if ext == "" or cleanExt.contains(ext): if not ignore.contains(name): echo "removing: ", path @@ -302,99 +319,14 @@ proc clean(args: string) = echo "removing dir: ", path removeDir(path) -# -------------- update ------------------------------------------------------- - -when defined(withUpdate): - when defined(windows): - {.warning: "Windows users: Make sure to run 'koch update' in Bash.".} - - proc update(args: string) = - when defined(windows): - echo("Windows users: Make sure to be running this in Bash. ", - "If you aren't, press CTRL+C now.") - - var thisDir = getAppDir() - var git = findExe("git") - echo("Checking for git repo and git executable...") - if existsDir(thisDir & "/.git") and git != "": - echo("Git repo found!") - # use git to download latest source - echo("Checking for updates...") - discard startCmd(git & " fetch origin master") - var procs = startCmd(git & " diff origin/master master") - var errcode = procs.waitForExit() - var output = readLine(procs.outputStream) - echo(output) - if errcode == 0: - if output == "": - # No changes - echo("No update. Exiting...") - return - else: - echo("Fetching updates from repo...") - var pullout = execCmdEx(git & " pull origin master") - if pullout[1] != 0: - quit("An error has occurred.") - else: - if pullout[0].startsWith("Already up-to-date."): - quit("No new changes fetched from the repo. " & - "Local branch must be ahead of it. Exiting...") - else: - quit("An error has occurred.") - - else: - echo("No repo or executable found!") - when defined(haveZipLib): - echo("Falling back.. Downloading source code from repo...") - # use dom96's httpclient to download zip - downloadFile("https://github.com/Araq/Nim/zipball/master", - thisDir / "update.zip") - try: - echo("Extracting source code from archive...") - var zip: TZipArchive - discard open(zip, thisDir & "/update.zip", fmRead) - extractAll(zip, thisDir & "/") - except: - quit("Error reading archive.") - else: - quit("No failback available. Exiting...") - - echo("Starting update...") - boot(args) - echo("Update complete!") - # -------------- builds a release --------------------------------------------- -proc run7z(platform: string, patterns: varargs[string]) = - const tmpDir = "nim-" & VersionAsString - createDir tmpDir - try: - for pattern in patterns: - for f in walkFiles(pattern): - if "nimcache" notin f: - copyFile(f, tmpDir / f) - exec("7z a -tzip $1-$2.zip $1" % [tmpDir, platform]) - finally: - removeDir tmpDir - proc winRelease() = - boot(" -d:release") - #buildTool("tools/niminst/niminst", " -d:release") - buildTool("tools/nimgrep", " -d:release") - buildTool("compiler/nimfix/nimfix", " -d:release") - buildTool("compiler/nimsuggest/nimsuggest", " -d:release") - - #run7z("win32", "bin/nim.exe", "bin/c2nim.exe", "bin/nimgrep.exe", - # "bin/nimfix.exe", - # "bin/nimble.exe", "bin/*.dll", - # "config", "dist/*.dll", "examples", "lib", - # "readme.txt", "contributors.txt", "copying.txt") - - # second step: XXX build 64 bit version + exec(r"call ci\nsis_build.bat " & VersionAsString) # -------------- tests -------------------------------------------------------- -template `|`(a, b): expr = (if a.len > 0: a else: b) +template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = # we compile the tester with taintMode:on to have a basic @@ -444,14 +376,9 @@ of cmdArgument: of "xz": xz(op.cmdLineRest) of "nsis": nsis(op.cmdLineRest) of "geninstall": geninstall(op.cmdLineRest) - of "install": install(op.cmdLineRest) + of "distrohelper": geninstall() of "testinstall": testUnixInstall() of "test", "tests": tests(op.cmdLineRest) - of "update": - when defined(withUpdate): - update(op.cmdLineRest) - else: - quit "this Koch has not been compiled with -d:withUpdate" of "temp": temp(op.cmdLineRest) of "winrelease": winRelease() else: showHelp() diff --git a/lib/arch/arch.nim b/lib/arch/arch.nim index a11bfb21f..c8ae3da1c 100644 --- a/lib/arch/arch.nim +++ b/lib/arch/arch.nim @@ -6,6 +6,9 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # +# Architecture-specific optimizations and features. +# arch.nim can be imported by only a subset of the +# architectures supported by Nim. when defined(windows): const diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 4296cb0ae..19452b4a8 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -333,7 +333,7 @@ proc parseStmt*(s: string): NimNode {.noSideEffect, compileTime.} = let x = internalErrorFlag() if x.len > 0: raise newException(ValueError, x) -proc getAst*(macroOrTemplate: expr): NimNode {.magic: "ExpandToAst", noSideEffect.} +proc getAst*(macroOrTemplate: untyped): NimNode {.magic: "ExpandToAst", noSideEffect.} ## Obtains the AST nodes returned from a macro or template invocation. ## Example: ## @@ -342,7 +342,7 @@ proc getAst*(macroOrTemplate: expr): NimNode {.magic: "ExpandToAst", noSideEffec ## macro FooMacro() = ## var ast = getAst(BarTemplate()) -proc quote*(bl: stmt, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} +proc quote*(bl: typed, op = "``"): NimNode {.magic: "QuoteAst", noSideEffect.} ## Quasi-quoting operator. ## Accepts an expression or a block and returns the AST that represents it. ## Within the quoted AST, you are able to interpolate NimNode expressions @@ -663,7 +663,7 @@ proc copyChildrenTo*(src, dest: NimNode) {.compileTime.}= for i in 0 .. < src.len: dest.add src[i].copyNimTree -template expectRoutine(node: NimNode): stmt = +template expectRoutine(node: NimNode) = expectKind(node, RoutineNodes) proc name*(someProc: NimNode): NimNode {.compileTime.} = @@ -870,7 +870,7 @@ proc hasArgOfName* (params: NimNode; name: string): bool {.compiletime.}= ## Search nnkFormalParams for an argument. assert params.kind == nnkFormalParams for i in 1 .. <params.len: - template node: expr = params[i] + template node: untyped = params[i] if name.eqIdent( $ node[0]): return true @@ -891,7 +891,7 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = else: n == bindSym"true" # hacky solution for now when not defined(booting): - template emit*(e: static[string]): stmt {.deprecated.} = + template emit*(e: static[string]): untyped {.deprecated.} = ## accepts a single string argument and treats it as nim code ## that should be inserted verbatim in the program ## Example: @@ -900,6 +900,6 @@ when not defined(booting): ## emit("echo " & '"' & "hello world".toUpper & '"') ## ## Deprecated since version 0.15 since it's so rarely useful. - macro payload: stmt {.gensym.} = + macro payload: untyped {.gensym.} = result = parseStmt(e) payload() diff --git a/lib/impure/re.nim b/lib/impure/re.nim index bf397550a..bd86bcdcf 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -359,7 +359,7 @@ proc parallelReplace*(s: string, subs: openArray[ proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Raises ``EIO`` if an + ## `parallelReplace`) and writes back to `outfile`. Raises ``IOError`` if an ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile).string writeFile(outfile, x.parallelReplace(subs)) diff --git a/lib/nimbase.h b/lib/nimbase.h index 37f8f5c3d..52de60969 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -415,16 +415,6 @@ struct TFrame { #define NIM_POSIX_INIT __attribute__((constructor)) -#if defined(_MSCVER) && defined(__i386__) -__declspec(naked) int __fastcall NimXadd(volatile int* pNum, int val) { - __asm { - lock xadd dword ptr [ECX], EDX - mov EAX, EDX - ret - } -} -#endif - #ifdef __GNUC__ # define likely(x) __builtin_expect(x, 1) # define unlikely(x) __builtin_expect(x, 0) diff --git a/lib/packages/docutils/rstgen.nim b/lib/packages/docutils/rstgen.nim index 8b5bb0e8f..47247dd7c 100644 --- a/lib/packages/docutils/rstgen.nim +++ b/lib/packages/docutils/rstgen.nim @@ -415,18 +415,28 @@ proc sortIndex(a: var openArray[IndexEntry]) = a[j] <- v if h == 1: break +proc escapeLink(s: string): string = + result = newStringOfCap(s.len + s.len shr 2) + for c in items(s): + case c + of 'a'..'z', '_', 'A'..'Z', '0'..'9', '.', '#', ',', '/': + result.add c + else: + add(result, "%") + add(result, toHex(ord(c), 2)) + proc generateSymbolIndex(symbols: seq[IndexEntry]): string = - result = "" + result = "<dl>" var i = 0 while i < symbols.len: - let keyword= symbols[i].keyword - let cleaned_keyword = keyword[1..keyword.high - 1] - result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><ul class=\"simple\"><dd>\n", + let keyword = symbols[i].keyword + let cleaned_keyword = keyword.escapeLink + result.addf("<dt><a name=\"$2\" href=\"#$2\"><span>$1:</span></a></dt><dd><ul class=\"simple\">\n", [keyword, cleaned_keyword]) var j = i while j < symbols.len and keyword == symbols[j].keyword: let - url = symbols[j].link + url = symbols[j].link.escapeLink text = if not symbols[j].linkTitle.isNil: symbols[j].linkTitle else: url desc = if not symbols[j].linkDesc.isNil: symbols[j].linkDesc else: "" if desc.len > 0: @@ -439,6 +449,7 @@ proc generateSymbolIndex(symbols: seq[IndexEntry]): string = inc j result.add("</ul></dd>\n") i = j + result.add("</dl>") proc isDocumentationTitle(hyperlink: string): bool = ## Returns true if the hyperlink is actually a documentation title. @@ -463,9 +474,9 @@ proc indentToLevel(level: var int, newLevel: int): string = if level == newLevel: return if newLevel > level: - result = repeat("<ul>", newLevel - level) + result = repeat("<li><ul>", newLevel - level) else: - result = repeat("</ul>", level - newLevel) + result = repeat("</ul></li>", level - newLevel) level = newLevel proc generateDocumentationTOC(entries: seq[IndexEntry]): string = @@ -503,7 +514,7 @@ proc generateDocumentationTOC(entries: seq[IndexEntry]): string = else: result.add(level.indentToLevel(levels[L].level)) result.add("<li><a href=\"" & link & "\">" & - levels[L].text & "</a>\n") + levels[L].text & "</a></li>\n") inc L result.add(level.indentToLevel(1) & "</ul>\n") assert(not titleRef.isNil, @@ -520,7 +531,7 @@ proc generateDocumentationIndex(docs: IndexedDocs): string = for title in titles: let tocList = generateDocumentationTOC(docs.getOrDefault(title)) result.add("<ul><li><a href=\"" & - title.link & "\">" & title.keyword & "</a>\n" & tocList & "</ul>\n") + title.link & "\">" & title.keyword & "</a>\n" & tocList & "</li></ul>\n") proc generateDocumentationJumps(docs: IndexedDocs): string = ## Returns a plain list of hyperlinks to documentation TOCs in HTML. @@ -783,7 +794,7 @@ proc renderImage(d: PDoc, n: PRstNode, result: var string) = if s.valid: dispA(d.target, options, " align=\"$1\"", "", [strip(s)]) if options.len > 0: options = dispF(d.target, "$1", "[$1]", [options]) - + if arg.valid: let htmlOut = if isObject: "<object data=\"$1\" type=\"image/svg+xml\"$2 >" & content & "</object>" diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index bd69b2328..b96e88b6c 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -1694,6 +1694,8 @@ var INADDR_ANY* {.importc, header: "<netinet/in.h>".}: InAddrScalar ## IPv4 local host address. + INADDR_LOOPBACK* {.importc, header: "<netinet/in.h>".}: InAddrScalar + ## IPv4 loopback address. INADDR_BROADCAST* {.importc, header: "<netinet/in.h>".}: InAddrScalar ## IPv4 broadcast address. @@ -2653,6 +2655,12 @@ proc poll*(a1: ptr TPollfd, a2: Tnfds, a3: int): cint {. proc realpath*(name, resolved: cstring): cstring {. importc: "realpath", header: "<stdlib.h>".} +proc mkstemp*(tmpl: cstring): cint {.importc, header: "<stdlib.h>".} + ## Create a temporary file. + ## + ## **Warning**: The `tmpl` argument is written to by `mkstemp` and thus + ## can't be a string literal. If in doubt copy the string before passing it. + proc utimes*(path: cstring, times: ptr array[2, Timeval]): int {. importc: "utimes", header: "<sys/time.h>".} ## Sets file access and modification times. diff --git a/lib/posix/termios.nim b/lib/posix/termios.nim index c3934c6a9..1fbccba9c 100644 --- a/lib/posix/termios.nim +++ b/lib/posix/termios.nim @@ -259,3 +259,13 @@ proc tcFlow*(fd: cint; action: cint): cint {.importc: "tcflow", # Get process group ID for session leader for controlling terminal FD. proc tcGetSid*(fd: cint): Pid {.importc: "tcgetsid", header: "<termios.h>".} + +# Window size ioctl. Should work on on any Unix that xterm has been ported to. +var TIOCGWINSZ*{.importc, header: "<sys/ioctl.h>".}: culong + +type IOctl_WinSize* {.importc: "struct winsize", header: "<sys/ioctl.h>", + final, pure.} = object + ws_row*, ws_col*, ws_xpixel*, ws_ypixel*: cushort + +proc ioctl*(fd: cint, request: culong, reply: ptr IOctl_WinSize): int {. + importc: "ioctl", header: "<stdio.h>", varargs.} diff --git a/lib/pure/algorithm.nim b/lib/pure/algorithm.nim index b83daf245..739cdc16b 100644 --- a/lib/pure/algorithm.nim +++ b/lib/pure/algorithm.nim @@ -179,7 +179,7 @@ proc sort*[T](a: var openArray[T], ## sort(myStrArray, system.cmp) ## ## You can inline adhoc comparison procs with the `do notation - ## <manual.html#do-notation>`_. Example: + ## <manual.html#procedures-do-notation>`_. Example: ## ## .. code-block:: nim ## diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 79bc1b96d..8336da1fb 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -9,7 +9,7 @@ include "system/inclrtl" -import os, oids, tables, strutils, macros, times, heapqueue +import os, oids, tables, strutils, times, heapqueue import nativesockets, net, queues @@ -105,6 +105,36 @@ export Port, SocketFlag ## be used to await a procedure returning a ``Future[void]``: ## ``await socket.send("foobar")``. ## +## If an awaited future completes with an error, then ``await`` will re-raise +## this error. To avoid this, you can use the ``yield`` keyword instead of +## ``await``. The following section shows different ways that you can handle +## exceptions in async procs. +## +## Handling Exceptions +## ~~~~~~~~~~~~~~~~~~~ +## +## The most reliable way to handle exceptions is to use ``yield`` on a future +## then check the future's ``failed`` property. For example: +## +## .. code-block:: Nim +## var future = sock.recv(100) +## yield future +## if future.failed: +## # Handle exception +## +## The ``async`` procedures also offer limited support for the try statement. +## +## .. code-block:: Nim +## try: +## let data = await sock.recv(100) +## echo("Received ", data) +## except: +## # Handle exception +## +## Unfortunately the semantics of the try statement may not always be correct, +## and occassionally the compilation may fail altogether. +## As such it is better to use the former style when possible. +## ## Discarding futures ## ------------------ ## @@ -126,279 +156,10 @@ export Port, SocketFlag ## * Can't await in a ``except`` body ## * Forward declarations for async procs are broken, ## link includes workaround: https://github.com/nim-lang/Nim/issues/3182. -## * FutureVar[T] needs to be completed manually. # TODO: Check if yielded future is nil and throw a more meaningful exception -# -- Futures - -type - FutureBase* = ref object of RootObj ## Untyped future. - cb: proc () {.closure,gcsafe.} - finished: bool - error*: ref Exception ## Stored exception - errorStackTrace*: string - when not defined(release): - stackTrace: string ## For debugging purposes only. - id: int - fromProc: string - - Future*[T] = ref object of FutureBase ## Typed future. - value: T ## Stored value - - FutureVar*[T] = distinct Future[T] - - FutureError* = object of Exception - cause*: FutureBase - -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - -when not defined(release): - var currentID = 0 - -proc callSoon*(cbproc: proc ()) {.gcsafe.} - -proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = - ## Creates a new future. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - new(result) - result.finished = false - when not defined(release): - result.stackTrace = getStackTrace() - result.id = currentID - result.fromProc = fromProc - currentID.inc() - -proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = - ## Create a new ``FutureVar``. This Future type is ideally suited for - ## situations where you want to avoid unnecessary allocations of Futures. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - result = FutureVar[T](newFuture[T](fromProc)) - -proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. - Future[T](future).finished = false - Future[T](future).error = nil - -proc checkFinished[T](future: Future[T]) = - ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. - when not defined(release): - if future.finished: - var msg = "" - msg.add("An attempt was made to complete a Future more than once. ") - msg.add("Details:") - msg.add("\n Future ID: " & $future.id) - msg.add("\n Created in proc: " & future.fromProc) - msg.add("\n Stack trace to moment of creation:") - msg.add("\n" & indent(future.stackTrace.strip(), 4)) - when T is string: - msg.add("\n Contents (string): ") - msg.add("\n" & indent(future.value.repr, 4)) - msg.add("\n Stack trace to moment of secondary completion:") - msg.add("\n" & indent(getStackTrace().strip(), 4)) - var err = newException(FutureError, msg) - err.cause = future - raise err - -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. - template fut: expr = Future[T](future) - checkFinished(fut) - assert(fut.error == nil) - fut.finished = true - if fut.cb != nil: - fut.cb() - -proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - future.finished = true - future.error = error - future.errorStackTrace = - if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) - if future.cb != nil: - future.cb() - else: - # This is to prevent exceptions from being silently ignored when a future - # is discarded. - # TODO: This may turn out to be a bad idea. - # Turns out this is a bad idea. - #raise error - discard - -proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: You most likely want the other ``callback`` setter which - ## passes ``future`` as a param to the callback. - future.cb = cb - if future.finished: - callSoon(future.cb) - -proc `callback=`*[T](future: Future[T], - cb: proc (future: Future[T]) {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.callback = proc () = cb(future) - -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) - -proc read*[T](future: Future[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: - injectStacktrace(future) - raise future.error - when T isnot void: - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. - ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if future.error != nil: return future.error - else: - raise newException(ValueError, "No error in future.") - -proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. - ## - ## Unlike ``read``, this function will not raise an exception if the - ## Future has not been finished. - result = Future[T](future).value - -proc finished*[T](future: Future[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - future.finished - -proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. - return future.error != nil - -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - future.callback = - proc () = - if future.failed: - injectStacktrace(future) - raise future.error - -proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`and`") - fut1.callback = - proc () = - if fut2.finished: retFuture.complete() - fut2.callback = - proc () = - if fut1.finished: retFuture.complete() - return retFuture - -proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`or`") - proc cb() = - if not retFuture.finished: retFuture.complete() - fut1.callback = cb - fut2.callback = cb - return retFuture - -proc all*[T](futs: varargs[Future[T]]): auto = - ## Returns a future which will complete once - ## all futures in ``futs`` complete. - ## - ## If the awaited futures are not ``Future[void]``, the returned future - ## will hold the values of all awaited futures in a sequence. - ## - ## If the awaited futures *are* ``Future[void]``, - ## this proc returns ``Future[void]``. - - when T is void: - var - retFuture = newFuture[void]("asyncdispatch.all") - completedFutures = 0 - - let totalFutures = len(futs) - - for fut in futs: - fut.callback = proc(f: Future[T]) = - inc(completedFutures) - - if completedFutures == totalFutures: - retFuture.complete() - - return retFuture - - else: - var - retFuture = newFuture[seq[T]]("asyncdispatch.all") - retValues = newSeq[T](len(futs)) - completedFutures = 0 - - for i, fut in futs: - proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = - retValues[i] = f.read() - inc(completedFutures) - - if completedFutures == len(retValues): - retFuture.complete(retValues) - - setCallback(i) - - return retFuture +include includes/asyncfutures type PDispatcherBase = ref object of RootRef @@ -500,48 +261,49 @@ when defined(windows) or defined(nimdoc): raise newException(ValueError, "No handles or timers registered in dispatcher.") - let at = p.adjustedTimeout(timeout) - var llTimeout = - if at == -1: winlean.INFINITE - else: at.int32 - - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: + if p.handles.len != 0: + let at = p.adjustedTimeout(timeout) + var llTimeout = + if at == -1: winlean.INFINITE + else: at.int32 + + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. if customOverlapped.data.cell.data != nil: system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) # Timer processing. processTimers(p) @@ -749,8 +511,8 @@ when defined(windows) or defined(nimdoc): retFuture.complete("") return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, - flags = {SocketFlag.SafeDisconn}): Future[int] = + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] = ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must ## at least be of that size. Returned future will complete once all the ## data requested is read, a part of the data has been read, or the socket @@ -773,7 +535,7 @@ when defined(windows) or defined(nimdoc): #buf[] = '\0' var dataBuf: TWSABuf - dataBuf.buf = buf + dataBuf.buf = cast[cstring](buf) dataBuf.len = size.ULONG var bytesReceived: Dword @@ -819,16 +581,18 @@ when defined(windows) or defined(nimdoc): retFuture.complete(bytesReceived) return retFuture - proc send*(socket: AsyncFD, data: string, + proc send*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[void] = - ## Sends ``data`` to ``socket``. The returned future will complete once all + ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all ## data has been sent. + ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, you must use GC_ref/GC_unref calls + ## to avoid early freeing of the buffer verifyPresence(socket) var retFuture = newFuture[void]("send") var dataBuf: TWSABuf - dataBuf.buf = data # since this is not used in a callback, this is fine - dataBuf.len = data.len.ULONG + dataBuf.buf = cast[cstring](buf) + dataBuf.len = size.ULONG var bytesReceived, lowFlags: Dword var ol = PCustomOverlapped() @@ -1281,43 +1045,45 @@ else: proc poll*(timeout = 500) = let p = getGlobalDispatcher() - for info in p.selector.select(p.adjustedTimeout(timeout)): - let data = PData(info.key.data) - assert data.fd == info.key.fd.AsyncFD - #echo("In poll ", data.fd.cint) - # There may be EvError here, but we handle them in callbacks, - # so that exceptions can be raised from `send(...)` and - # `recv(...)` routines. - - if EvRead in info.events: - # Callback may add items to ``data.readCBs`` which causes issues if - # we are iterating over ``data.readCBs`` at the same time. We therefore - # make a copy to iterate over. - let currentCBs = data.readCBs - data.readCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.readCBs.add(cb) - - if EvWrite in info.events: - let currentCBs = data.writeCBs - data.writeCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.writeCBs.add(cb) - - if info.key in p.selector: - var newEvents: set[Event] - if data.readCBs.len != 0: newEvents = {EvRead} - if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite} - if newEvents != info.key.events: - update(data.fd, newEvents) - else: - # FD no longer a part of the selector. Likely been closed - # (e.g. socket disconnected). - discard + + if p.selector.len > 0: + for info in p.selector.select(p.adjustedTimeout(timeout)): + let data = PData(info.key.data) + assert data.fd == info.key.fd.AsyncFD + #echo("In poll ", data.fd.cint) + # There may be EvError here, but we handle them in callbacks, + # so that exceptions can be raised from `send(...)` and + # `recv(...)` routines. + + if EvRead in info.events: + # Callback may add items to ``data.readCBs`` which causes issues if + # we are iterating over ``data.readCBs`` at the same time. We therefore + # make a copy to iterate over. + let currentCBs = data.readCBs + data.readCBs = @[] + for cb in currentCBs: + if not cb(data.fd): + # Callback wants to be called again. + data.readCBs.add(cb) + + if EvWrite in info.events: + let currentCBs = data.writeCBs + data.writeCBs = @[] + for cb in currentCBs: + if not cb(data.fd): + # Callback wants to be called again. + data.writeCBs.add(cb) + + if info.key in p.selector: + var newEvents: set[Event] + if data.readCBs.len != 0: newEvents = {EvRead} + if data.writeCBs.len != 0: newEvents = newEvents + {EvWrite} + if newEvents != info.key.events: + update(data.fd, newEvents) + else: + # FD no longer a part of the selector. Likely been closed + # (e.g. socket disconnected). + discard # Timer processing. processTimers(p) @@ -1398,7 +1164,7 @@ else: addRead(socket, cb) return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[int] = var retFuture = newFuture[int]("recvInto") @@ -1422,7 +1188,7 @@ else: addRead(socket, cb) return retFuture - proc send*(socket: AsyncFD, data: string, + proc send*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[void] = var retFuture = newFuture[void]("send") @@ -1430,8 +1196,8 @@ else: proc cb(sock: AsyncFD): bool = result = true - let netSize = data.len-written - var d = data.cstring + let netSize = size-written + var d = cast[cstring](buf) let res = send(sock.SocketHandle, addr d[written], netSize.cint, MSG_NOSIGNAL) if res < 0: @@ -1576,368 +1342,31 @@ proc accept*(socket: AsyncFD, retFut.complete(future.read.client) return retFut -# -- Await Macro +proc send*(socket: AsyncFD, data: string, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``data`` to ``socket``. The returned future will complete once all + ## data has been sent. + var retFuture = newFuture[void]("send") -proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = - # Skips a nest of StmtList's. - result = node - if node[0].kind == nnkStmtList: - result = skipUntilStmtList(node[0]) - -proc skipStmtList(node: NimNode): NimNode {.compileTime.} = - result = node - if node[0].kind == nnkStmtList: - result = node[0] - -template createCb(retFutureSym, iteratorNameSym, - name: untyped) = - var nameIterVar = iteratorNameSym - #{.push stackTrace: off.} - proc cb {.closure,gcsafe.} = - try: - if not nameIterVar.finished: - var next = nameIterVar() - if next == nil: - assert retFutureSym.finished, "Async procedure's (" & - name & ") return Future was not finished." - else: - next.callback = cb - except: - if retFutureSym.finished: - # Take a look at tasyncexceptions for the bug which this fixes. - # That test explains it better than I can here. - raise - else: - retFutureSym.fail(getCurrentException()) - cb() - #{.pop.} -proc generateExceptionCheck(futSym, - tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = - if tryStmt.kind == nnkNilLit: - result = rootReceiver - else: - var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] - let errorNode = newDotExpr(futSym, newIdentNode("error")) - for i in 1 .. <tryStmt.len: - let exceptBranch = tryStmt[i] - if exceptBranch[0].kind == nnkStmtList: - exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) - else: - var exceptIdentCount = 0 - var ifCond: NimNode - for i in 0 .. <exceptBranch.len: - let child = exceptBranch[i] - if child.kind == nnkIdent: - let cond = infix(errorNode, "of", child) - if exceptIdentCount == 0: - ifCond = cond - else: - ifCond = infix(ifCond, "or", cond) - else: - break - exceptIdentCount.inc - - expectKind(exceptBranch[exceptIdentCount], nnkStmtList) - exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) - # -> -> else: raise futSym.error - exceptionChecks.add((newIdentNode("true"), - newNimNode(nnkRaiseStmt).add(errorNode))) - # Read the future if there is no error. - # -> else: futSym.read - let elseNode = newNimNode(nnkElse, fromNode) - elseNode.add newNimNode(nnkStmtList, fromNode) - elseNode[0].add rootReceiver - - let ifBody = newStmtList() - ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) - ifBody.add newIfStmt(exceptionChecks) - ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) - - result = newIfStmt( - (newDotExpr(futSym, newIdentNode("failed")), ifBody) - ) - result.add elseNode - -template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, - rootReceiver: expr, fromNode: NimNode) = - ## Params: - ## futureVarNode: The NimNode which is a symbol identifying the Future[T] - ## variable to yield. - ## fromNode: Used for better debug information (to give context). - ## valueReceiver: The node which defines an expression that retrieves the - ## future's value. - ## - ## rootReceiver: ??? TODO - # -> yield future<x> - result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) - # -> future<x>.read - valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) - result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, - fromNode) - -template createVar(result: var NimNode, futSymName: string, - asyncProc: NimNode, - valueReceiver, rootReceiver: expr, - fromNode: NimNode) = - result = newNimNode(nnkStmtList, fromNode) - var futSym = genSym(nskVar, "future") - result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - useVar(result, futSym, valueReceiver, rootReceiver, fromNode) - -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - tryStmt: NimNode): NimNode {.compileTime.} = - #echo(node.treeRepr) - result = node - case node.kind - of nnkReturnStmt: - result = newNimNode(nnkStmtList, node) - if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) - else: - result.add newCall(newIdentNode("complete"), retFutureSym) - else: - let x = node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt) - if x.kind == nnkYieldStmt: result.add x - else: - result.add newCall(newIdentNode("complete"), retFutureSym, x) - - result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) - return # Don't process the children of this return stmt - of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": - case node[1].kind - of nnkIdent, nnkInfix, nnkDotExpr: - # await x - # await x or y - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x - of nnkCall, nnkCommand: - # await foo(p, x) - # await foo p, x - var futureValue: NimNode - result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, - futureValue, node) + var copiedData = data + GC_ref(copiedData) # we need to protect data until send operation is completed + # or failed. + + let sendFut = socket.send(addr copiedData[0], data.len, flags) + sendFut.callback = + proc () = + GC_unref(copiedData) + if sendFut.failed: + retFuture.fail(sendFut.error) else: - error("Invalid node kind in 'await', got: " & $node[1].kind) - elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": - # foo await x - var newCommand = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], - newCommand, node) - - of nnkVarSection, nnkLetSection: - case node[0][2].kind - of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": - # var x = await y - var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & $node[0][0].ident, node[0][2][1], - newVarSection[0][2], newVarSection, node) - else: discard - of nnkAsgn: - case node[1].kind - of nnkCommand: - if node[1][0].ident == !"await": - # x = await y - var newAsgn = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) - else: discard - of nnkDiscardStmt: - # discard await x - if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": - var newDiscard = node - result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], - newDiscard[0], newDiscard, node) - of nnkTryStmt: - # try: await x; except: ... - result = newNimNode(nnkStmtList, node) - template wrapInTry(n, tryBody: expr) = - var temp = n - n[0] = tryBody - tryBody = temp - - # Transform ``except`` body. - # TODO: Could we perform some ``await`` transformation here to get it - # working in ``except``? - tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) - - proc processForTry(n: NimNode, i: var int, - res: NimNode): bool {.compileTime.} = - ## Transforms the body of the tryStmt. Does not transform the - ## body in ``except``. - ## Returns true if the tryStmt node was transformed into an ifStmt. - result = false - var skipped = n.skipStmtList() - while i < skipped.len: - var processed = processBody(skipped[i], retFutureSym, - subTypeIsVoid, n) - - # Check if we transformed the node into an exception check. - # This suggests skipped[i] contains ``await``. - if processed.kind != skipped[i].kind or processed.len != skipped[i].len: - processed = processed.skipUntilStmtList() - expectKind(processed, nnkStmtList) - expectKind(processed[2][1], nnkElse) - i.inc - - if not processForTry(n, i, processed[2][1][0]): - # We need to wrap the nnkElse nodes back into a tryStmt. - # As they are executed if an exception does not happen - # inside the awaited future. - # The following code will wrap the nodes inside the - # original tryStmt. - wrapInTry(n, processed[2][1][0]) - - res.add processed - result = true - else: - res.add skipped[i] - i.inc - var i = 0 - if not processForTry(node, i, result): - # If the tryStmt hasn't been transformed we can just put the body - # back into it. - wrapInTry(node, result) - return - else: discard - - for i in 0 .. <result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) - -proc getName(node: NimNode): string {.compileTime.} = - case node.kind - of nnkPostfix: - return $node[1].ident - of nnkIdent: - return $node.ident - of nnkEmpty: - return "anonymous" - else: - error("Unknown name.") - -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = - ## This macro transforms a single procedure into a closure iterator. - ## The ``async`` macro supports a stmtList holding multiple async procedures. - if prc.kind notin {nnkProcDef, nnkLambda}: - error("Cannot transform this node kind into an async proc." & - " Proc definition or lambda node expected.") - - hint("Processing " & prc[0].getName & " as an async proc.") - - let returnType = prc[3][0] - var baseType: NimNode - # Verify that the return type is a Future[T] - if returnType.kind == nnkBracketExpr: - let fut = repr(returnType[0]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[1] - elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": - let fut = repr(returnType[1]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[2] - elif returnType.kind == nnkEmpty: - baseType = returnType - else: - error("Expected return type of 'Future' got '" & repr(returnType) & "'") - - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].ident == !"void") - - var outerProcBody = newNimNode(nnkStmtList, prc[6]) - - # -> var retFuture = newFuture[T]() - var retFutureSym = genSym(nskVar, "retFuture") - var subRetType = - if returnType.kind == nnkEmpty: newIdentNode("void") - else: baseType - outerProcBody.add( - newVarStmt(retFutureSym, - newCall( - newNimNode(nnkBracketExpr, prc[6]).add( - newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. - subRetType), - newLit(prc[0].getName)))) # Get type from return type of this proc - - # -> iterator nameIter(): FutureBase {.closure.} = - # -> {.push warning[resultshadowed]: off.} - # -> var result: T - # -> {.pop.} - # -> <proc_body> - # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - # don't do anything with forward bodies (empty) - if procBody.kind != nnkEmpty: - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) - - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = getAst createCb(retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb - - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) - - result = prc - - # Remove the 'async' pragma. - for i in 0 .. <result[4].len: - if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": - result[4].del(i) - result[4] = newEmptyNode() - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result[3][0] = parseExpr("Future[void]") - if procBody.kind != nnkEmpty: - result[6] = outerProcBody - #echo(treeRepr(result)) - #if prc[0].getName == "testInfix": - # echo(toStrLit(result)) - -macro async*(prc: untyped): untyped = - ## Macro which processes async procedures into the appropriate - ## iterators and yield statements. - if prc.kind == nnkStmtList: - for oneProc in prc: - result = newStmtList() - result.add asyncSingleProc(oneProc) - else: - result = asyncSingleProc(prc) - when defined(nimDumpAsync): - echo repr result + retFuture.complete() + + return retFuture -proc recvLine*(socket: AsyncFD): Future[string] {.async.} = +# -- Await Macro +include asyncmacro + +proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} = ## Reads a line of data from ``socket``. Returned future will complete once ## a full line is read or an error occurs. ## @@ -1955,6 +1384,8 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## ## **Note**: This procedure is mostly used for testing. You likely want to ## use ``asyncnet.recvLine`` instead. + ## + ## **Deprecated since version 0.15.0**: Use ``asyncnet.recvLine()`` instead. template addNLIfEmpty(): stmt = if result.len == 0: diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index 5df606ea8..ffe6a391e 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -24,6 +24,8 @@ import asyncdispatch, os +# TODO: Fix duplication introduced by PR #4683. + when defined(windows) or defined(nimdoc): import winlean else: @@ -112,6 +114,80 @@ proc openAsync*(filename: string, mode = fmRead): AsyncFile = register(result.fd) +proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = + ## Read ``size`` bytes from the specified file asynchronously starting at + ## the current position of the file pointer. + ## + ## If the file pointer is past the end of the file then an empty string is + ## returned. + var retFuture = newFuture[int]("asyncfile.readBuffer") + + when defined(windows) or defined(nimdoc): + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: f.fd, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount > 0 + assert bytesCount <= size + f.offset.inc bytesCount + retFuture.complete(bytesCount) + else: + if errcode.int32 == ERROR_HANDLE_EOF: + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + ol.offset = DWord(f.offset and 0xffffffff) + ol.offsetHigh = DWord(f.offset shr 32) + + # According to MSDN we're supposed to pass nil to lpNumberOfBytesRead. + let ret = readFile(f.fd.Handle, buf, size.int32, nil, + cast[POVERLAPPED](ol)) + if not ret.bool: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + var bytesRead: DWord + let overlappedRes = getOverlappedResult(f.fd.Handle, + cast[POverlapped](ol), bytesRead, false.WinBool) + if not overlappedRes.bool: + let err = osLastError() + if err.int32 == ERROR_HANDLE_EOF: + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + else: + assert bytesRead > 0 + assert bytesRead <= size + f.offset.inc bytesRead + retFuture.complete(bytesRead) + else: + proc cb(fd: AsyncFD): bool = + result = true + let res = read(fd.cint, cast[cstring](buf), size.cint) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EAGAIN: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + elif res == 0: + # EOF + retFuture.complete(0) + else: + f.offset.inc(res) + retFuture.complete(res) + + if not cb(f.fd): + addRead(f.fd, cb) + + return retFuture + proc read*(f: AsyncFile, size: int): Future[string] = ## Read ``size`` bytes from the specified file asynchronously starting at ## the current position of the file pointer. @@ -238,6 +314,73 @@ proc readAll*(f: AsyncFile): Future[string] {.async.} = return result.add data +proc writeBuffer*(f: AsyncFile, buf: pointer, size: int): Future[void] = + ## Writes ``size`` bytes from ``buf`` to the file specified asynchronously. + ## + ## The returned Future will complete once all data has been written to the + ## specified file. + var retFuture = newFuture[void]("asyncfile.writeBuffer") + when defined(windows) or defined(nimdoc): + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: f.fd, cb: + proc (fd: AsyncFD, bytesCount: DWord, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + assert bytesCount == size.int32 + f.offset.inc(size) + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + ol.offset = DWord(f.offset and 0xffffffff) + ol.offsetHigh = DWord(f.offset shr 32) + + # According to MSDN we're supposed to pass nil to lpNumberOfBytesWritten. + let ret = writeFile(f.fd.Handle, buf, size.int32, nil, + cast[POVERLAPPED](ol)) + if not ret.bool: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + # Request completed immediately. + var bytesWritten: DWord + let overlappedRes = getOverlappedResult(f.fd.Handle, + cast[POverlapped](ol), bytesWritten, false.WinBool) + if not overlappedRes.bool: + retFuture.fail(newException(OSError, osErrorMsg(osLastError()))) + else: + assert bytesWritten == size.int32 + f.offset.inc(size) + retFuture.complete() + else: + var written = 0 + + proc cb(fd: AsyncFD): bool = + result = true + let remainderSize = size-written + var cbuf = cast[cstring](buf) + let res = write(fd.cint, addr cbuf[written], remainderSize.cint) + if res < 0: + let lastError = osLastError() + if lastError.int32 != EAGAIN: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc res + f.offset.inc res + if res != remainderSize: + result = false # We still have data to write. + else: + retFuture.complete() + + if not cb(f.fd): + addWrite(f.fd, cb) + return retFuture + proc write*(f: AsyncFile, data: string): Future[void] = ## Writes ``data`` to the file specified asynchronously. ## diff --git a/lib/pure/asyncftpclient.nim b/lib/pure/asyncftpclient.nim index fd899e080..019a18f55 100644 --- a/lib/pure/asyncftpclient.nim +++ b/lib/pure/asyncftpclient.nim @@ -281,7 +281,7 @@ proc getFile(ftp: AsyncFtpClient, file: File, total: BiggestInt, assertReply(await(ftp.expectReply()), "226") proc defaultOnProgressChanged*(total, progress: BiggestInt, - speed: float): Future[void] {.nimcall,gcsafe.} = + speed: float): Future[void] {.nimcall,gcsafe,procvar.} = ## Default FTP ``onProgressChanged`` handler. Does nothing. result = newFuture[void]() #echo(total, " ", progress, " ", speed) @@ -354,6 +354,20 @@ proc store*(ftp: AsyncFtpClient, file, dest: string, await doUpload(ftp, destFile, onProgressChanged) +proc rename*(ftp: AsyncFtpClient, nameFrom: string, nameTo: string) {.async.} = + ## Rename a file or directory on the remote FTP Server from current name + ## ``name_from`` to new name ``name_to`` + assertReply(await ftp.send("RNFR " & name_from), "350") + assertReply(await ftp.send("RNTO " & name_to), "250") + +proc removeFile*(ftp: AsyncFtpClient, filename: string) {.async.} = + ## Delete a file ``filename`` on the remote FTP server + assertReply(await ftp.send("DELE " & filename), "250") + +proc removeDir*(ftp: AsyncFtpClient, dir: string) {.async.} = + ## Delete a directory ``dir`` on the remote FTP server + assertReply(await ftp.send("RMD " & dir), "250") + proc newAsyncFtpClient*(address: string, port = Port(21), user, pass = ""): AsyncFtpClient = ## Creates a new ``AsyncFtpClient`` object. @@ -373,6 +387,11 @@ when not defined(testing) and isMainModule: echo await ftp.listDirs() await ftp.store("payload.jpg", "payload.jpg") await ftp.retrFile("payload.jpg", "payload2.jpg") + await ftp.rename("payload.jpg", "payload_renamed.jpg") + await ftp.store("payload.jpg", "payload_remove.jpg") + await ftp.removeFile("payload_remove.jpg") + await ftp.createDir("deleteme") + await ftp.removeDir("deleteme") echo("Finished") waitFor main(ftp) diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index 6a7326e83..a658097f9 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -9,6 +9,12 @@ ## This module implements a high performance asynchronous HTTP server. ## +## This HTTP server has not been designed to be used in production, but +## for testing applications locally. Because of this, when deploying your +## application you should use a reverse proxy (for example nginx) instead of +## allowing users to connect directly to this server. +## +## ## Examples ## -------- ## @@ -38,7 +44,7 @@ export httpcore except parseHeader type Request* = object client*: AsyncSocket # TODO: Separate this into a Response object? - reqMethod*: string + reqMethod*: HttpMethod headers*: HttpHeaders protocol*: tuple[orig: string, major, minor: int] url*: Uri @@ -127,7 +133,14 @@ proc processClient(client: AsyncSocket, address: string, var i = 0 for linePart in lineFut.mget.split(' '): case i - of 0: request.reqMethod.shallowCopy(linePart.normalize) + of 0: + try: + # TODO: this is likely slow. + request.reqMethod = parseEnum[HttpMethod]("http" & linePart) + except ValueError: + asyncCheck request.respond(Http400, "Invalid request method. Got: " & + linePart) + continue of 1: parseUri(linePart, request.url) of 2: try: @@ -159,7 +172,7 @@ proc processClient(client: AsyncSocket, address: string, request.client.close() return - if request.reqMethod == "post": + if request.reqMethod == HttpPost: # Check for Expect header if request.headers.hasKey("Expect"): if "100-continue" in request.headers["Expect"]: @@ -178,17 +191,12 @@ proc processClient(client: AsyncSocket, address: string, else: request.body = await client.recv(contentLength) assert request.body.len == contentLength - elif request.reqMethod == "post": + elif request.reqMethod == HttpPost: await request.respond(Http400, "Bad Request. No Content-Length.") continue - case request.reqMethod - of "get", "post", "head", "put", "delete", "trace", "options", - "connect", "patch": - await callback(request) - else: - await request.respond(Http400, "Invalid request method. Got: " & - request.reqMethod) + # Call the user's callback. + await callback(request) if "upgrade" in request.headers.getOrDefault("connection"): return diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim new file mode 100644 index 000000000..3d004e84c --- /dev/null +++ b/lib/pure/asyncmacro.nim @@ -0,0 +1,515 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## AsyncMacro +## ************* +## `asyncdispatch` module depends on the `asyncmacro` module to work properly. + +import macros, strutils + +proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = + # Skips a nest of StmtList's. + result = node + if node[0].kind == nnkStmtList: + result = skipUntilStmtList(node[0]) + +proc skipStmtList(node: NimNode): NimNode {.compileTime.} = + result = node + if node[0].kind == nnkStmtList: + result = node[0] + +template createCb(retFutureSym, iteratorNameSym, + name, futureVarCompletions: untyped) = + var nameIterVar = iteratorNameSym + #{.push stackTrace: off.} + proc cb {.closure,gcsafe.} = + try: + if not nameIterVar.finished: + var next = nameIterVar() + if next == nil: + assert retFutureSym.finished, "Async procedure's (" & + name & ") return Future was not finished." + else: + next.callback = cb + except: + if retFutureSym.finished: + # Take a look at tasyncexceptions for the bug which this fixes. + # That test explains it better than I can here. + raise + else: + retFutureSym.fail(getCurrentException()) + + futureVarCompletions + cb() + #{.pop.} +proc generateExceptionCheck(futSym, + tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = + if tryStmt.kind == nnkNilLit: + result = rootReceiver + else: + var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] + let errorNode = newDotExpr(futSym, newIdentNode("error")) + for i in 1 .. <tryStmt.len: + let exceptBranch = tryStmt[i] + if exceptBranch[0].kind == nnkStmtList: + exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) + else: + var exceptIdentCount = 0 + var ifCond: NimNode + for i in 0 .. <exceptBranch.len: + let child = exceptBranch[i] + if child.kind == nnkIdent: + let cond = infix(errorNode, "of", child) + if exceptIdentCount == 0: + ifCond = cond + else: + ifCond = infix(ifCond, "or", cond) + else: + break + exceptIdentCount.inc + + expectKind(exceptBranch[exceptIdentCount], nnkStmtList) + exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) + # -> -> else: raise futSym.error + exceptionChecks.add((newIdentNode("true"), + newNimNode(nnkRaiseStmt).add(errorNode))) + # Read the future if there is no error. + # -> else: futSym.read + let elseNode = newNimNode(nnkElse, fromNode) + elseNode.add newNimNode(nnkStmtList, fromNode) + elseNode[0].add rootReceiver + + let ifBody = newStmtList() + ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) + ifBody.add newIfStmt(exceptionChecks) + ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) + + result = newIfStmt( + (newDotExpr(futSym, newIdentNode("failed")), ifBody) + ) + result.add elseNode + +template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, + rootReceiver: expr, fromNode: NimNode) = + ## Params: + ## futureVarNode: The NimNode which is a symbol identifying the Future[T] + ## variable to yield. + ## fromNode: Used for better debug information (to give context). + ## valueReceiver: The node which defines an expression that retrieves the + ## future's value. + ## + ## rootReceiver: ??? TODO + # -> yield future<x> + result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) + # -> future<x>.read + valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) + result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, + fromNode) + +template createVar(result: var NimNode, futSymName: string, + asyncProc: NimNode, + valueReceiver, rootReceiver: expr, + fromNode: NimNode) = + result = newNimNode(nnkStmtList, fromNode) + var futSym = genSym(nskVar, "future") + result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y + useVar(result, futSym, valueReceiver, rootReceiver, fromNode) + +proc createFutureVarCompletions(futureVarIdents: seq[NimNode]): NimNode + {.compileTime.} = + result = newStmtList() + # Add calls to complete each FutureVar parameter. + for ident in futureVarIdents: + # Only complete them if they have not been completed already by the user. + result.add newIfStmt( + ( + newCall(newIdentNode("not"), + newDotExpr(ident, newIdentNode("finished"))), + newCall(newIdentNode("complete"), ident) + ) + ) + +proc processBody(node, retFutureSym: NimNode, + subTypeIsVoid: bool, futureVarIdents: seq[NimNode], + tryStmt: NimNode): NimNode {.compileTime.} = + #echo(node.treeRepr) + result = node + case node.kind + of nnkReturnStmt: + result = newNimNode(nnkStmtList, node) + if node[0].kind == nnkEmpty: + if not subTypeIsVoid: + result.add newCall(newIdentNode("complete"), retFutureSym, + newIdentNode("result")) + else: + result.add newCall(newIdentNode("complete"), retFutureSym) + else: + let x = node[0].processBody(retFutureSym, subTypeIsVoid, + futureVarIdents, tryStmt) + if x.kind == nnkYieldStmt: result.add x + else: + result.add newCall(newIdentNode("complete"), retFutureSym, x) + + result.add createFutureVarCompletions(futureVarIdents) + + result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) + return # Don't process the children of this return stmt + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + case node[1].kind + of nnkIdent, nnkInfix, nnkDotExpr, nnkCall, nnkCommand: + # await x + # await x or y + # await foo(p, x) + # await foo p, x + var futureValue: NimNode + result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, + futureValue, node) + else: + error("Invalid node kind in 'await', got: " & $node[1].kind) + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + var newCommand = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], + newCommand, node) + + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + var newVarSection = node # TODO: Should this use copyNimNode? + result.createVar("future" & $node[0][0].ident, node[0][2][1], + newVarSection[0][2], newVarSection, node) + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + var newAsgn = node + result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + var newDiscard = node + result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], + newDiscard[0], newDiscard, node) + of nnkTryStmt: + # try: await x; except: ... + result = newNimNode(nnkStmtList, node) + template wrapInTry(n, tryBody: expr) = + var temp = n + n[0] = tryBody + tryBody = temp + + # Transform ``except`` body. + # TODO: Could we perform some ``await`` transformation here to get it + # working in ``except``? + tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, + futureVarIdents, nil) + + proc processForTry(n: NimNode, i: var int, + res: NimNode): bool {.compileTime.} = + ## Transforms the body of the tryStmt. Does not transform the + ## body in ``except``. + ## Returns true if the tryStmt node was transformed into an ifStmt. + result = false + var skipped = n.skipStmtList() + while i < skipped.len: + var processed = processBody(skipped[i], retFutureSym, + subTypeIsVoid, futureVarIdents, n) + + # Check if we transformed the node into an exception check. + # This suggests skipped[i] contains ``await``. + if processed.kind != skipped[i].kind or processed.len != skipped[i].len: + processed = processed.skipUntilStmtList() + expectKind(processed, nnkStmtList) + expectKind(processed[2][1], nnkElse) + i.inc + + if not processForTry(n, i, processed[2][1][0]): + # We need to wrap the nnkElse nodes back into a tryStmt. + # As they are executed if an exception does not happen + # inside the awaited future. + # The following code will wrap the nodes inside the + # original tryStmt. + wrapInTry(n, processed[2][1][0]) + + res.add processed + result = true + else: + res.add skipped[i] + i.inc + var i = 0 + if not processForTry(node, i, result): + # If the tryStmt hasn't been transformed we can just put the body + # back into it. + wrapInTry(node, result) + return + else: discard + + for i in 0 .. <result.len: + result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, + futureVarIdents, nil) + +proc getName(node: NimNode): string {.compileTime.} = + case node.kind + of nnkPostfix: + return $node[1].ident + of nnkIdent: + return $node.ident + of nnkEmpty: + return "anonymous" + else: + error("Unknown name.") + +proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} = + result = @[] + for i in 1 .. <len(params): + expectKind(params[i], nnkIdentDefs) + if params[i][1].kind == nnkBracketExpr and + ($params[i][1][0].ident).normalize == "futurevar": + result.add(params[i][0]) + +proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = + ## This macro transforms a single procedure into a closure iterator. + ## The ``async`` macro supports a stmtList holding multiple async procedures. + if prc.kind notin {nnkProcDef, nnkLambda}: + error("Cannot transform this node kind into an async proc." & + " Proc definition or lambda node expected.") + + hint("Processing " & prc[0].getName & " as an async proc.") + + let returnType = prc[3][0] + var baseType: NimNode + # Verify that the return type is a Future[T] + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[1] + elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": + let fut = repr(returnType[1]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + baseType = returnType[2] + elif returnType.kind == nnkEmpty: + baseType = returnType + else: + error("Expected return type of 'Future' got '" & repr(returnType) & "'") + + let subtypeIsVoid = returnType.kind == nnkEmpty or + (baseType.kind == nnkIdent and returnType[1].ident == !"void") + + let futureVarIdents = getFutureVarIdents(prc[3]) + + var outerProcBody = newNimNode(nnkStmtList, prc[6]) + + # -> var retFuture = newFuture[T]() + var retFutureSym = genSym(nskVar, "retFuture") + var subRetType = + if returnType.kind == nnkEmpty: newIdentNode("void") + else: baseType + outerProcBody.add( + newVarStmt(retFutureSym, + newCall( + newNimNode(nnkBracketExpr, prc[6]).add( + newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. + subRetType), + newLit(prc[0].getName)))) # Get type from return type of this proc + + # -> iterator nameIter(): FutureBase {.closure.} = + # -> {.push warning[resultshadowed]: off.} + # -> var result: T + # -> {.pop.} + # -> <proc_body> + # -> complete(retFuture, result) + var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") + var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, + futureVarIdents, nil) + # don't do anything with forward bodies (empty) + if procBody.kind != nnkEmpty: + if not subtypeIsVoid: + procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), + newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( + newIdentNode("warning"), newIdentNode("resultshadowed")), + newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} + + procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( + newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T + + procBody.insert(2, newNimNode(nnkPragma).add( + newIdentNode("pop"))) # -> {.pop.}) + + procBody.add( + newCall(newIdentNode("complete"), + retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) + else: + # -> complete(retFuture) + procBody.add(newCall(newIdentNode("complete"), retFutureSym)) + + procBody.add(createFutureVarCompletions(futureVarIdents)) + + var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], + procBody, nnkIteratorDef) + closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) + outerProcBody.add(closureIterator) + + # -> createCb(retFuture) + #var cbName = newIdentNode("cb") + var procCb = getAst createCb(retFutureSym, iteratorNameSym, + newStrLitNode(prc[0].getName), + createFutureVarCompletions(futureVarIdents)) + outerProcBody.add procCb + + # -> return retFuture + outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) + + result = prc + + # Remove the 'async' pragma. + for i in 0 .. <result[4].len: + if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": + result[4].del(i) + result[4] = newEmptyNode() + if subtypeIsVoid: + # Add discardable pragma. + if returnType.kind == nnkEmpty: + # Add Future[void] + result[3][0] = parseExpr("Future[void]") + if procBody.kind != nnkEmpty: + result[6] = outerProcBody + #echo(treeRepr(result)) + #if prc[0].getName == "testInfix": + # echo(toStrLit(result)) + +macro async*(prc: untyped): untyped = + ## Macro which processes async procedures into the appropriate + ## iterators and yield statements. + if prc.kind == nnkStmtList: + for oneProc in prc: + result = newStmtList() + result.add asyncSingleProc(oneProc) + else: + result = asyncSingleProc(prc) + when defined(nimDumpAsync): + echo repr result + + +# Multisync +proc emptyNoop[T](x: T): T = + # The ``await``s are replaced by a call to this for simplicity. + when T isnot void: + return x + +proc stripAwait(node: NimNode): NimNode = + ## Strips out all ``await`` commands from a procedure body, replaces them + ## with ``emptyNoop`` for simplicity. + result = node + + let emptyNoopSym = bindSym("emptyNoop") + + case node.kind + of nnkCommand, nnkCall: + if node[0].kind == nnkIdent and node[0].ident == !"await": + node[0] = emptyNoopSym + elif node.len > 1 and node[1].kind == nnkCommand and + node[1][0].kind == nnkIdent and node[1][0].ident == !"await": + # foo await x + node[1][0] = emptyNoopSym + of nnkVarSection, nnkLetSection: + case node[0][2].kind + of nnkCommand: + if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": + # var x = await y + node[0][2][0] = emptyNoopSym + else: discard + of nnkAsgn: + case node[1].kind + of nnkCommand: + if node[1][0].ident == !"await": + # x = await y + node[1][0] = emptyNoopSym + else: discard + of nnkDiscardStmt: + # discard await x + if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and + node[0][0].ident == !"await": + node[0][0] = emptyNoopSym + else: discard + + for i in 0 .. <result.len: + result[i] = stripAwait(result[i]) + +proc splitParams(param: NimNode, async: bool): NimNode = + expectKind(param, nnkIdentDefs) + result = param + if param[1].kind == nnkInfix and $param[1][0].ident in ["|", "or"]: + let firstType = param[1][1] + let firstTypeName = $firstType.ident + let secondType = param[1][2] + let secondTypeName = $secondType.ident + + # Make sure that at least one has the name `async`, otherwise we shouldn't + # touch it. + if not ("async" in firstTypeName.normalize or + "async" in secondTypeName.normalize): + return + + if async: + if firstTypeName.normalize.startsWith("async"): + result = newIdentDefs(param[0], param[1][1]) + elif secondTypeName.normalize.startsWith("async"): + result = newIdentDefs(param[0], param[1][2]) + else: + if not firstTypeName.normalize.startsWith("async"): + result = newIdentDefs(param[0], param[1][1]) + elif not secondTypeName.normalize.startsWith("async"): + result = newIdentDefs(param[0], param[1][2]) + +proc stripReturnType(returnType: NimNode): NimNode = + # Strip out the 'Future' from 'Future[T]'. + result = returnType + if returnType.kind == nnkBracketExpr: + let fut = repr(returnType[0]) + if fut != "Future": + error("Expected return type of 'Future' got '" & fut & "'") + result = returnType[1] + +proc splitProc(prc: NimNode): (NimNode, NimNode) = + ## Takes a procedure definition which takes a generic union of arguments, + ## for example: proc (socket: Socket | AsyncSocket). + ## It transforms them so that ``proc (socket: Socket)`` and + ## ``proc (socket: AsyncSocket)`` are returned. + result[0] = prc.copyNimTree() + result[0][3][0] = stripReturnType(result[0][3][0]) + for i in 1 .. <result[0][3].len: + result[0][3][i] = splitParams(result[0][3][i], false) + result[0][6] = stripAwait(result[0][6]) + + result[1] = prc.copyNimTree() + for i in 1 .. <result[1][3].len: + result[1][3][i] = splitParams(result[1][3][i], true) + +macro multisync*(prc: untyped): untyped = + ## Macro which processes async procedures into both asynchronous and + ## synchronous procedures. + ## + ## The generated async procedures use the ``async`` macro, whereas the + ## generated synchronous procedures simply strip off the ``await`` calls. + hint("Processing " & prc[0].getName & " as a multisync proc.") + + let (sync, asyncPrc) = splitProc(prc) + result = newStmtList() + result.add(asyncSingleProc(asyncPrc)) + result.add(sync) diff --git a/lib/pure/asyncnet.nim b/lib/pure/asyncnet.nim index a1988f4a6..3b64c278f 100644 --- a/lib/pure/asyncnet.nim +++ b/lib/pure/asyncnet.nim @@ -62,6 +62,8 @@ import os export SOBool +# TODO: Remove duplication introduced by PR #4683. + const defineSsl = defined(ssl) or defined(nimdoc) when defineSsl: @@ -157,15 +159,23 @@ when defineSsl: await socket.fd.AsyncFd.send(data, flags) proc appeaseSsl(socket: AsyncSocket, flags: set[SocketFlag], - sslError: cint) {.async.} = + sslError: cint): Future[bool] {.async.} = + ## Returns ``true`` if ``socket`` is still connected, otherwise ``false``. + result = true case sslError of SSL_ERROR_WANT_WRITE: await sendPendingSslData(socket, flags) of SSL_ERROR_WANT_READ: var data = await recv(socket.fd.AsyncFD, BufferSize, flags) - let ret = bioWrite(socket.bioIn, addr data[0], data.len.cint) - if ret < 0: - raiseSSLError() + let length = len(data) + if length > 0: + let ret = bioWrite(socket.bioIn, addr data[0], data.len.cint) + if ret < 0: + raiseSSLError() + elif length == 0: + # connection not properly closed by remote side or connection dropped + SSL_set_shutdown(socket.sslHandle, SSL_RECEIVED_SHUTDOWN) + result = false else: raiseSSLError("Cannot appease SSL.") @@ -173,13 +183,27 @@ when defineSsl: op: expr) = var opResult {.inject.} = -1.cint while opResult < 0: + # Call the desired operation. opResult = op # Bit hackish here. # TODO: Introduce an async template transformation pragma? + + # Send any remaining pending SSL data. yield sendPendingSslData(socket, flags) + + # If the operation failed, try to see if SSL has some data to read + # or write. if opResult < 0: let err = getSslError(socket.sslHandle, opResult.cint) - yield appeaseSsl(socket, flags, err.cint) + let fut = appeaseSsl(socket, flags, err.cint) + yield fut + if not fut.read(): + # Socket disconnected. + if SocketFlag.SafeDisconn in flags: + break + else: + raiseSSLError("Socket has been disconnected") + proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = ## Connects ``socket`` to server at ``address:port``. @@ -193,7 +217,7 @@ proc connect*(socket: AsyncSocket, address: string, port: Port) {.async.} = sslSetConnectState(socket.sslHandle) sslLoop(socket, flags, sslDoHandshake(socket.sslHandle)) -template readInto(buf: cstring, size: int, socket: AsyncSocket, +template readInto(buf: pointer, size: int, socket: AsyncSocket, flags: set[SocketFlag]): int = ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. Note that ## this is a template and not a proc. @@ -202,10 +226,10 @@ template readInto(buf: cstring, size: int, socket: AsyncSocket, when defineSsl: # SSL mode. sslLoop(socket, flags, - sslRead(socket.sslHandle, buf, size.cint)) + sslRead(socket.sslHandle, cast[cstring](buf), size.cint)) res = opResult else: - var recvIntoFut = recvInto(socket.fd.AsyncFD, buf, size, flags) + var recvIntoFut = asyncdispatch.recvInto(socket.fd.AsyncFD, buf, size, flags) yield recvIntoFut # Not in SSL mode. res = recvIntoFut.read() @@ -218,6 +242,54 @@ template readIntoBuf(socket: AsyncSocket, socket.bufLen = size size +proc recvInto*(socket: AsyncSocket, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): Future[int] {.async.} = + ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``. + ## + ## For buffered sockets this function will attempt to read all the requested + ## data. It will read this data in ``BufferSize`` chunks. + ## + ## For unbuffered sockets this function makes no effort to read + ## all the data requested. It will return as much data as the operating system + ## gives it. + ## + ## If socket is disconnected during the + ## recv operation then the future may complete with only a part of the + ## requested data. + ## + ## If socket is disconnected and no data is available + ## to be read then the future will complete with a value of ``0``. + if socket.isBuffered: + let originalBufPos = socket.currPos + + if socket.bufLen == 0: + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) + if res == 0: + return 0 + + var read = 0 + var cbuf = cast[cstring](buf) + while read < size: + if socket.currPos >= socket.bufLen: + if SocketFlag.Peek in flags: + # We don't want to get another buffer if we're peeking. + break + let res = socket.readIntoBuf(flags - {SocketFlag.Peek}) + if res == 0: + break + + let chunk = min(socket.bufLen-socket.currPos, size-read) + copyMem(addr(cbuf[read]), addr(socket.buffer[socket.currPos]), chunk) + read.inc(chunk) + socket.currPos.inc(chunk) + + if SocketFlag.Peek in flags: + # Restore old buffer cursor position. + socket.currPos = originalBufPos + result = read + else: + result = readInto(buf, size, socket, flags) + proc recv*(socket: AsyncSocket, size: int, flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = ## Reads **up to** ``size`` bytes from ``socket``. @@ -270,6 +342,19 @@ proc recv*(socket: AsyncSocket, size: int, let read = readInto(addr result[0], size, socket, flags) result.setLen(read) +proc send*(socket: AsyncSocket, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}) {.async.} = + ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all + ## data has been sent. + assert socket != nil + if socket.isSsl: + when defineSsl: + sslLoop(socket, flags, + sslWrite(socket.sslHandle, cast[cstring](buf), size.cint)) + await sendPendingSslData(socket, flags) + else: + await send(socket.fd.AsyncFD, buf, size, flags) + proc send*(socket: AsyncSocket, data: string, flags = {SocketFlag.SafeDisconn}) {.async.} = ## Sends ``data`` to ``socket``. The returned future will complete once all @@ -320,7 +405,7 @@ proc accept*(socket: AsyncSocket, return retFut proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], - flags = {SocketFlag.SafeDisconn}) {.async.} = + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {.async.} = ## Reads a line of data from ``socket`` into ``resString``. ## ## If a full line is read ``\r\L`` is not @@ -333,13 +418,14 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], ## is read) then line will be set to ``""``. ## The partial line **will be lost**. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## ## **Warning**: ``recvLineInto`` on unbuffered sockets assumes that the ## protocol uses ``\r\L`` to delimit a new line. - ## - ## **Warning**: ``recvLineInto`` currently uses a raw pointer to a string for - ## performance reasons. This will likely change soon to use FutureVars. assert SocketFlag.Peek notin flags ## TODO: assert(not resString.mget.isNil(), "String inside resString future needs to be initialised") @@ -386,6 +472,12 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], else: resString.mget.add socket.buffer[socket.currPos] socket.currPos.inc() + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) else: var c = "" while true: @@ -407,10 +499,17 @@ proc recvLineInto*(socket: AsyncSocket, resString: FutureVar[string], resString.complete() return resString.mget.add c + + # Verify that this isn't a DOS attack: #3847. + if resString.mget.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) resString.complete() proc recvLine*(socket: AsyncSocket, - flags = {SocketFlag.SafeDisconn}): Future[string] {.async.} = + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once ## a full line is read or an error occurs. ## @@ -424,6 +523,10 @@ proc recvLine*(socket: AsyncSocket, ## is read) then line will be set to ``""``. ## The partial line **will be lost**. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: The ``Peek`` flag is not yet implemented. ## ## **Warning**: ``recvLine`` on unbuffered sockets assumes that the protocol @@ -433,7 +536,7 @@ proc recvLine*(socket: AsyncSocket, # TODO: Optimise this var resString = newFutureVar[string]("asyncnet.recvLine") resString.mget() = "" - await socket.recvLineInto(resString, flags) + await socket.recvLineInto(resString, flags, maxLength) result = resString.mget() proc listen*(socket: AsyncSocket, backlog = SOMAXCONN) {.tags: [ReadIOEffect].} = diff --git a/lib/pure/basic2d.nim b/lib/pure/basic2d.nim index 7d74424fa..e4696c6a8 100644 --- a/lib/pure/basic2d.nim +++ b/lib/pure/basic2d.nim @@ -117,13 +117,13 @@ proc safeArccos(v:float):float= template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors + ## implements binary operators ``+``, ``-``, ``*`` and ``/`` for vectors proc s*(a,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a.x,b.x),s(a.y,b.y)) proc s*(a:Vector2d,b:float):Vector2d {.inline,noInit.} = vector2d(s(a.x,b),s(a.y,b)) proc s*(a:float,b:Vector2d):Vector2d {.inline,noInit.} = vector2d(s(a,b.x),s(a,b.y)) template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors + ## implements inplace binary operators ``+=``, ``-=``, ``/=`` and ``*=`` for vectors proc s*(a:var Vector2d,b:Vector2d) {.inline.} = s(a.x,b.x) ; s(a.y,b.y) proc s*(a:var Vector2d,b:float) {.inline.} = s(a.x,b) ; s(a.y,b) diff --git a/lib/pure/basic3d.nim b/lib/pure/basic3d.nim index 424c191f8..f7a9c237c 100644 --- a/lib/pure/basic3d.nim +++ b/lib/pure/basic3d.nim @@ -117,7 +117,6 @@ proc safeArccos(v:float):float= return arccos(clamp(v,-1.0,1.0)) template makeBinOpVector(s:expr)= - ## implements binary operators + , - , * and / for vectors proc s*(a,b:Vector3d):Vector3d {.inline,noInit.} = vector3d(s(a.x,b.x),s(a.y,b.y),s(a.z,b.z)) proc s*(a:Vector3d,b:float):Vector3d {.inline,noInit.} = @@ -126,11 +125,10 @@ template makeBinOpVector(s:expr)= vector3d(s(a,b.x),s(a,b.y),s(a,b.z)) template makeBinOpAssignVector(s:expr)= - ## implements inplace binary operators += , -= , /= and *= for vectors proc s*(a:var Vector3d,b:Vector3d) {.inline.} = - s(a.x,b.x) ; s(a.y,b.y) ; s(a.z,b.z) + s(a.x,b.x); s(a.y,b.y); s(a.z,b.z) proc s*(a:var Vector3d,b:float) {.inline.} = - s(a.x,b) ; s(a.y,b) ; s(a.z,b) + s(a.x,b); s(a.y,b); s(a.z,b) diff --git a/lib/pure/collections/sharedtables.nim b/lib/pure/collections/sharedtables.nim index 17600b272..28509caa1 100644 --- a/lib/pure/collections/sharedtables.nim +++ b/lib/pure/collections/sharedtables.nim @@ -15,6 +15,8 @@ import hashes, math, locks +include "system/inclrtl" + type KeyValuePair[A, B] = tuple[hcode: Hash, key: A, val: B] KeyValuePairSeq[A, B] = ptr array[10_000_000, KeyValuePair[A, B]] @@ -35,9 +37,12 @@ proc enlarge[A, B](t: var SharedTable[A, B]) = t.dataLen = size swap(t.data, n) for i in 0..<oldSize: - if isFilled(n[i].hcode): - var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) - rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) + let eh = n[i].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + rawInsert(t, t.data, n[i].key, n[i].val, eh, j) deallocShared(n) template withLock(t, x: untyped) = @@ -47,7 +52,7 @@ template withLock(t, x: untyped) = template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body: untyped) = - ## retrieves the value at ``t[key]``. + ## retrieves the value at ``t[key]``. ## `value` can be modified in the scope of the ``withValue`` call. ## ## .. code-block:: nim @@ -55,7 +60,7 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, ## sharedTable.withValue(key, value) do: ## # block is executed only if ``key`` in ``t`` ## # value is threadsafe in block - ## value.name = "username" + ## value.name = "username" ## value.uid = 1000 ## acquire(t.lock) @@ -71,15 +76,15 @@ template withValue*[A, B](t: var SharedTable[A, B], key: A, template withValue*[A, B](t: var SharedTable[A, B], key: A, value, body1, body2: untyped) = - ## retrieves the value at ``t[key]``. + ## retrieves the value at ``t[key]``. ## `value` can be modified in the scope of the ``withValue`` call. - ## + ## ## .. code-block:: nim ## ## sharedTable.withValue(key, value) do: ## # block is executed only if ``key`` in ``t`` ## # value is threadsafe in block - ## value.name = "username" + ## value.name = "username" ## value.uid = 1000 ## do: ## # block is executed when ``key`` not in ``t`` diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index be3507137..a3dfd43a1 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -142,7 +142,8 @@ template delImpl() {.dirty.} = template clearImpl() {.dirty.} = for i in 0 .. <t.data.len: - t.data[i].hcode = 0 + when compiles(t.data[i].hcode): # CountTable records don't contain a hcode + t.data[i].hcode = 0 t.data[i].key = default(type(t.data[i].key)) t.data[i].val = default(type(t.data[i].val)) t.counter = 0 diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 9308095aa..778ea5ca3 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -118,7 +118,11 @@ template dataLen(t): untyped = len(t.data) include tableimpl -proc clear*[A, B](t: Table[A, B] | TableRef[A, B]) = +proc clear*[A, B](t: var Table[A, B]) = + ## Resets the table so that it is empty. + clearImpl() + +proc clear*[A, B](t: TableRef[A, B]) = ## Resets the table so that it is empty. clearImpl() @@ -270,9 +274,12 @@ proc enlarge[A, B](t: var Table[A, B]) = newSeq(n, len(t.data) * growthFactor) swap(t.data, n) for i in countup(0, high(n)): - if isFilled(n[i].hcode): - var j = -1 - rawGetKnownHC(t, n[i].key, n[i].hcode) - rawInsert(t, t.data, n[i].key, n[i].val, n[i].hcode, j) + let eh = n[i].hcode + if isFilled(eh): + var j: Hash = eh and maxHash(t) + while isFilled(t.data[j].hcode): + j = nextTry(j, maxHash(t)) + rawInsert(t, t.data, n[i].key, n[i].val, eh, j) proc mgetOrPut*[A, B](t: var Table[A, B], key: A, val: B): var B = ## retrieves value at ``t[key]`` or puts ``val`` if not present, either way @@ -457,7 +464,7 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -proc clear*[A, B](t: OrderedTable[A, B] | OrderedTableRef[A, B]) = +proc clear*[A, B](t: var OrderedTable[A, B] | OrderedTableRef[A, B]) = ## Resets the table so that it is empty. clearImpl() t.first = -1 @@ -786,7 +793,7 @@ proc len*[A](t: CountTable[A]): int = ## returns the number of keys in `t`. result = t.counter -proc clear*[A](t: CountTable[A] | CountTableRef[A]) = +proc clear*[A](t: var CountTable[A] | CountTableRef[A]) = ## Resets the table so that it is empty. clearImpl() t.counter = 0 diff --git a/lib/pure/concurrency/threadpool.nim b/lib/pure/concurrency/threadpool.nim index 9490dbbd5..8cdb83e19 100644 --- a/lib/pure/concurrency/threadpool.nim +++ b/lib/pure/concurrency/threadpool.nim @@ -297,9 +297,20 @@ var gSomeReady : Semaphore readyWorker: ptr Worker +# A workaround for recursion deadlock issue +# https://github.com/nim-lang/Nim/issues/4597 +var + numSlavesLock: Lock + numSlavesRunning {.guard: numSlavesLock}: int + numSlavesWaiting {.guard: numSlavesLock}: int + isSlave {.threadvar.}: bool + +numSlavesLock.initLock + gSomeReady.initSemaphore() proc slave(w: ptr Worker) {.thread.} = + isSlave = true while true: when declared(atomicStoreN): atomicStoreN(addr(w.ready), true, ATOMIC_SEQ_CST) @@ -311,7 +322,15 @@ proc slave(w: ptr Worker) {.thread.} = # XXX Somebody needs to look into this (why does this assertion fail # in Visual Studio?) when not defined(vcc): assert(not w.ready) + + withLock numSlavesLock: + inc numSlavesRunning + w.f(w, w.data) + + withLock numSlavesLock: + dec numSlavesRunning + if w.q.len != 0: w.cleanFlowVars if w.shutdown: w.shutdown = false @@ -464,10 +483,34 @@ proc nimSpawn3(fn: WorkerProc; data: pointer) {.compilerProc.} = fn(self, data) await(self.taskStarted) return - else: - await(gSomeReady) - else: - await(gSomeReady) + + if isSlave: + # Run under lock until `numSlavesWaiting` increment to avoid a + # race (otherwise two last threads might start waiting together) + withLock numSlavesLock: + if numSlavesRunning <= numSlavesWaiting + 1: + # All the other slaves are waiting + # If we wait now, we-re deadlocked until + # an external spawn happens ! + if currentPoolSize < maxPoolSize: + if not workersData[currentPoolSize].initialized: + activateWorkerThread(currentPoolSize) + let w = addr(workersData[currentPoolSize]) + atomicInc currentPoolSize + if selectWorker(w, fn, data): + return + else: + # There is no place in the pool. We're deadlocked. + # echo "Deadlock!" + discard + + inc numSlavesWaiting + + await(gSomeReady) + + if isSlave: + withLock numSlavesLock: + dec numSlavesWaiting var distinguishedLock: Lock diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 1fe0b297b..10a786555 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -452,7 +452,10 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode proc expected(x: var XmlParser, n: XmlNode): string = result = errorMsg(x, "</" & n.tag & "> expected") -template elemName(x: expr): expr = rawData(x) +template elemName(x: untyped): untyped = rawData(x) + +template adderr(x: untyped) = + errors.add(x) proc untilElementEnd(x: var XmlParser, result: XmlNode, errors: var seq[string]) = @@ -469,24 +472,24 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, # allow ``<p>`` in `<dd>`, `<dt>` and ``<li>`` in next case if htmlTag(x.elemName) in {tagLi, tagP, tagDt, tagDd, tagInput, tagOption}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagDd, tagDt, tagLi: if htmlTag(x.elemName) in {tagLi, tagDt, tagDd, tagInput, tagOption}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagTd, tagTh: if htmlTag(x.elemName) in {tagTr, tagTd, tagTh, tagTfoot, tagThead}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagTr: if htmlTag(x.elemName) == tagTr: - errors.add(expected(x, result)) + adderr(expected(x, result)) break of tagOptgroup: if htmlTag(x.elemName) in {tagOption, tagOptgroup}: - errors.add(expected(x, result)) + adderr(expected(x, result)) break else: discard result.addNode(parse(x, errors)) @@ -495,11 +498,11 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, next(x) else: #echo "5; expected: ", result.htmltag, " ", x.elemName - errors.add(expected(x, result)) + adderr(expected(x, result)) # do not skip it here! break of xmlEof: - errors.add(expected(x, result)) + adderr(expected(x, result)) break else: result.addNode(parse(x, errors)) @@ -516,14 +519,14 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = # we just ignore processing instructions for now next(x) of xmlError: - errors.add(errorMsg(x)) + adderr(errorMsg(x)) next(x) of xmlElementStart: result = newElement(toLowerAscii(x.elemName)) next(x) untilElementEnd(x, result, errors) of xmlElementEnd: - errors.add(errorMsg(x, "unexpected ending tag: " & x.elemName)) + adderr(errorMsg(x, "unexpected ending tag: " & x.elemName)) of xmlElementOpen: result = newElement(toLowerAscii(x.elemName)) next(x) @@ -537,16 +540,16 @@ proc parse(x: var XmlParser, errors: var seq[string]): XmlNode = next(x) break of xmlError: - errors.add(errorMsg(x)) + adderr(errorMsg(x)) next(x) break else: - errors.add(errorMsg(x, "'>' expected")) + adderr(errorMsg(x, "'>' expected")) next(x) break untilElementEnd(x, result, errors) of xmlAttribute, xmlElementClose: - errors.add(errorMsg(x, "<some_tag> expected")) + adderr(errorMsg(x, "<some_tag> expected")) next(x) of xmlCData: result = newCData(x.rawData) @@ -570,7 +573,7 @@ proc parseHtml*(s: Stream, filename: string, result = newElement("document") result.addNode(parse(x, errors)) #if x.kind != xmlEof: - # errors.add(errorMsg(x, "EOF expected")) + # adderr(errorMsg(x, "EOF expected")) while x.kind != xmlEof: var oldPos = x.bufpos # little hack to see if we made any progess result.addNode(parse(x, errors)) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 778ca2cbb..4404a9426 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -1,98 +1,130 @@ # # # Nim's Runtime Library -# (c) Copyright 2010 Dominik Picheta, Andreas Rumpf +# (c) Copyright 2016 Dominik Picheta, Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # ## This module implements a simple HTTP client that can be used to retrieve -## webpages/other data. -## -## -## **Note**: This module is not ideal, connection is not kept alive so sites with -## many redirects are expensive. As such in the future this module may change, -## and the current procedures will be deprecated. +## webpages and other data. ## ## Retrieving a website ## ==================== ## ## This example uses HTTP GET to retrieve -## ``http://google.com`` +## ``http://google.com``: ## ## .. code-block:: Nim +## var client = newHttpClient() ## echo(getContent("http://google.com")) ## +## The same action can also be performed asynchronously, simply use the +## ``AsyncHttpClient``: +## +## .. code-block:: Nim +## var client = newAsyncHttpClient() +## echo(await getContent("http://google.com")) +## +## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient`` +## is the same, so you can use whichever one suits you best in the examples +## shown here. +## +## **Note:** You will need to run asynchronous examples in an async proc +## otherwise you will get an ``Undeclared identifier: 'await'`` error. +## ## Using HTTP POST ## =============== ## ## This example demonstrates the usage of the W3 HTML Validator, it -## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to -## the server. +## uses ``multipart/form-data`` as the ``Content-Type`` to send the HTML to be +## validated to the server. ## ## .. code-block:: Nim +## var client = newHttpClient() ## var data = newMultipartData() ## data["output"] = "soap12" ## data["uploaded_file"] = ("test.html", "text/html", ## "<html><head></head><body><p>test</p></body></html>") ## -## echo postContent("http://validator.w3.org/check", multipart=data) +## echo client.postContent("http://validator.w3.org/check", multipart=data) ## -## Asynchronous HTTP requests -## ========================== +## Progress reporting +## ================== ## -## You simply have to create a new instance of the ``AsyncHttpClient`` object. -## You may then use ``await`` on the functions defined for that object. -## Keep in mind that the following code needs to be inside an asynchronous -## procedure. -## -## .. code-block::nim +## You may specify a callback procedure to be called during an HTTP request. +## This callback will be executed every second with information about the +## progress of the HTTP request. ## +## .. code-block:: Nim ## var client = newAsyncHttpClient() -## var resp = await client.request("http://google.com") +## proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = +## echo("Downloaded ", progress, " of ", total) +## echo("Current rate: ", speed div 1000, "kb/s") +## client.onProgressChanged = onProgressChanged +## discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") +## +## If you would like to remove the callback simply set it to ``nil``. +## +## .. code-block:: Nim +## client.onProgressChanged = nil ## ## SSL/TLS support ## =============== ## This requires the OpenSSL library, fortunately it's widely used and installed ## on many operating systems. httpclient will use SSL automatically if you give ## any of the functions a url with the ``https`` schema, for example: -## ``https://github.com/``, you also have to compile with ``ssl`` defined like so: +## ``https://github.com/``. +## +## You will also have to compile with ``ssl`` defined like so: ## ``nim c -d:ssl ...``. ## ## Timeouts ## ======== -## Currently all functions support an optional timeout, by default the timeout is set to -## `-1` which means that the function will never time out. The timeout is +## +## Currently only the synchronous functions support a timeout. +## The timeout is ## measured in milliseconds, once it is set any call on a socket which may -## block will be susceptible to this timeout, however please remember that the +## block will be susceptible to this timeout. +## +## It may be surprising but the ## function as a whole can take longer than the specified timeout, only ## individual internal calls on the socket are affected. In practice this means ## that as long as the server is sending data an exception will not be raised, -## if however data does not reach client within the specified timeout an ETimeout -## exception will then be raised. +## if however data does not reach the client within the specified timeout a +## ``TimeoutError`` exception will be raised. ## ## Proxy ## ===== ## -## A proxy can be specified as a param to any of these procedures, the ``newProxy`` -## constructor should be used for this purpose. However, -## currently only basic authentication is supported. +## A proxy can be specified as a param to any of the procedures defined in +## this module. To do this, use the ``newProxy`` constructor. Unfortunately, +## only basic authentication is supported at the moment. import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, - math, random, httpcore + math, random, httpcore, times import asyncnet, asyncdispatch import nativesockets export httpcore except parseHeader # TODO: The ``except`` doesn't work type - Response* = tuple[ - version: string, - status: string, - headers: HttpHeaders, - body: string] + Response* = object + version*: string + status*: string + headers*: HttpHeaders + body*: string + +proc code*(response: Response): HttpCode + {.raises: [ValueError, OverflowError].} = + ## Retrieves the specified response's ``HttpCode``. + ## + ## Raises a ``ValueError`` if the response's ``status`` does not have a + ## corresponding ``HttpCode``. + return response.status[0 .. 2].parseInt.HttpCode +type Proxy* = ref object url*: Uri auth*: string @@ -253,25 +285,6 @@ proc parseResponse(s: Socket, getBody: bool, timeout: int): Response = else: result.body = "" -type - HttpMethod* = enum ## the requested HttpMethod - httpHEAD, ## Asks for the response identical to the one that would - ## correspond to a GET request, but without the response - ## body. - httpGET, ## Retrieves the specified resource. - httpPOST, ## Submits data to be processed to the identified - ## resource. The data is included in the body of the - ## request. - httpPUT, ## Uploads a representation of the specified resource. - httpDELETE, ## Deletes the specified resource. - httpTRACE, ## Echoes back the received request, so that a client - ## can see what intermediate servers are adding or - ## changing in the request. - httpOPTIONS, ## Returns the HTTP methods that the server supports - ## for specified address. - httpCONNECT ## Converts the request connection to a transparent - ## TCP/IP tunnel, usually used for proxies. - {.deprecated: [THttpMethod: HttpMethod].} when not defined(ssl): @@ -389,15 +402,18 @@ proc format(p: MultipartData): tuple[header, body: string] = proc request*(url: string, httpMethod: string, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the custom method string specified by the ## | ``httpMethod`` parameter. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. var r = if proxy == nil: parseUri(url) else: proxy.url var hostUrl = if proxy == nil: r else: parseUri(url) - var headers = substr(httpMethod, len("http")) + var headers = httpMethod.toUpper() # TODO: Use generateHeaders further down once it supports proxies. var s = newSocket() @@ -481,15 +497,18 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", if body != "": s.send(body) - result = parseResponse(s, httpMethod != "httpHEAD", timeout) + result = parseResponse(s, httpMethod != "HEAD", timeout) proc request*(url: string, httpMethod = httpGET, extraHeaders = "", body = "", sslContext = defaultSSLContext, timeout = -1, - userAgent = defUserAgent, proxy: Proxy = nil): Response = + userAgent = defUserAgent, proxy: Proxy = nil): Response + {.deprecated.} = ## | Requests ``url`` with the specified ``httpMethod``. ## | Extra headers can be specified and must be separated by ``\c\L`` ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.request`` instead. result = request(url, $httpMethod, extraHeaders, body, sslContext, timeout, userAgent, proxy) @@ -512,12 +531,14 @@ proc getNewLocation(lastURL: string, headers: HttpHeaders): string = proc get*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): Response = + proxy: Proxy = nil): Response {.deprecated.} = ## | GETs the ``url`` and returns a ``Response`` object ## | This proc also handles redirection ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## ## **Deprecated since version 0.15.0**: use ``HttpClient.get`` instead. result = request(url, httpGET, extraHeaders, "", sslContext, timeout, userAgent, proxy) var lastURL = url @@ -531,12 +552,14 @@ proc get*(url: string, extraHeaders = "", maxRedirects = 5, proc getContent*(url: string, extraHeaders = "", maxRedirects = 5, sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, - proxy: Proxy = nil): string = + proxy: Proxy = nil): string {.deprecated.} = ## | GETs the body and returns it as a string. ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. ## | An optional timeout can be specified in milliseconds, if reading from the ## server takes longer than specified an ETimeout exception will be raised. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.getContent`` instead. var r = get(url, extraHeaders, maxRedirects, sslContext, timeout, userAgent, proxy) if r.status[0] in {'4','5'}: @@ -549,7 +572,7 @@ proc post*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): Response = + multipart: MultipartData = nil): Response {.deprecated.} = ## | POSTs ``body`` to the ``url`` and returns a ``Response`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. @@ -558,6 +581,8 @@ proc post*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.post`` instead. let (mpHeaders, mpBody) = format(multipart) template withNewLine(x): expr = @@ -587,7 +612,8 @@ proc postContent*(url: string, extraHeaders = "", body = "", sslContext: SSLContext = defaultSSLContext, timeout = -1, userAgent = defUserAgent, proxy: Proxy = nil, - multipart: MultipartData = nil): string = + multipart: MultipartData = nil): string + {.deprecated.} = ## | POSTs ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` ## | Extra headers can be specified and must be separated by ``\c\L``. @@ -595,6 +621,9 @@ proc postContent*(url: string, extraHeaders = "", body = "", ## server takes longer than specified an ETimeout exception will be raised. ## | The optional ``multipart`` parameter can be used to create ## ``multipart/form-data`` POSTs comfortably. + ## + ## **Deprecated since version 0.15.0**: use ``HttpClient.postContent`` + ## instead. var r = post(url, extraHeaders, body, maxRedirects, sslContext, timeout, userAgent, proxy, multipart) if r.status[0] in {'4','5'}: @@ -617,46 +646,116 @@ proc downloadFile*(url: string, outputFilename: string, else: fileError("Unable to open file") -proc generateHeaders(r: Uri, httpMethod: string, - headers: StringTableRef, body: string): string = - # TODO: Use this in the blocking HttpClient once it supports proxies. - result = substr(httpMethod, len("http")) - # TODO: Proxies +proc generateHeaders(requestUrl: Uri, httpMethod: string, + headers: HttpHeaders, body: string, proxy: Proxy): string = + # GET + result = httpMethod.toUpper() result.add ' ' - if r.path[0] != '/': result.add '/' - result.add(r.path) - if r.query.len > 0: - result.add("?" & r.query) + + if proxy.isNil: + # /path?query + if requestUrl.path[0] != '/': result.add '/' + result.add(requestUrl.path) + if requestUrl.query.len > 0: + result.add("?" & requestUrl.query) + else: + # Remove the 'http://' from the URL for CONNECT requests. + var modifiedUrl = requestUrl + modifiedUrl.scheme = "" + result.add($modifiedUrl) + + # HTTP/1.1\c\l result.add(" HTTP/1.1\c\L") - if r.port == "": - add(result, "Host: " & r.hostname & "\c\L") + # Host header. + if requestUrl.port == "": + add(result, "Host: " & requestUrl.hostname & "\c\L") else: - add(result, "Host: " & r.hostname & ":" & r.port & "\c\L") + add(result, "Host: " & requestUrl.hostname & ":" & requestUrl.port & "\c\L") - add(result, "Connection: Keep-Alive\c\L") + # Connection header. + if not headers.hasKey("Connection"): + add(result, "Connection: Keep-Alive\c\L") + + # Content length header. if body.len > 0 and not headers.hasKey("Content-Length"): add(result, "Content-Length: " & $body.len & "\c\L") + + # Proxy auth header. + if not proxy.isNil and proxy.auth != "": + let auth = base64.encode(proxy.auth, newline = "") + add(result, "Proxy-Authorization: basic " & auth & "\c\L") + for key, val in headers: add(result, key & ": " & val & "\c\L") add(result, "\c\L") type - AsyncHttpClient* = ref object - socket: AsyncSocket + ProgressChangedProc*[ReturnType] = + proc (total, progress, speed: BiggestInt): + ReturnType {.closure, gcsafe.} + + HttpClientBase*[SocketType] = ref object + socket: SocketType connected: bool currentURL: Uri ## Where we are currently connected. - headers*: StringTableRef + headers*: HttpHeaders ## Headers to send in requests. maxRedirects: int userAgent: string + timeout: int ## Only used for blocking HttpClient for now. + proxy: Proxy + ## ``nil`` or the callback to call when request progress changes. + when SocketType is Socket: + onProgressChanged*: ProgressChangedProc[void] + else: + onProgressChanged*: ProgressChangedProc[Future[void]] when defined(ssl): sslContext: net.SslContext + contentTotal: BiggestInt + contentProgress: BiggestInt + oneSecondProgress: BiggestInt + lastProgressReport: float + +type + HttpClient* = HttpClientBase[Socket] + +proc newHttpClient*(userAgent = defUserAgent, + maxRedirects = 5, sslContext = defaultSslContext, proxy: Proxy = nil, + timeout = -1): HttpClient = + ## Creates a new HttpClient instance. + ## + ## ``userAgent`` specifies the user agent that will be used when making + ## requests. + ## + ## ``maxRedirects`` specifies the maximum amount of redirects to follow, + ## default is 5. + ## + ## ``sslContext`` specifies the SSL context to use for HTTPS requests. + ## + ## ``proxy`` specifies an HTTP proxy to use for this HTTP client's + ## connections. + ## + ## ``timeout`` specifies the number of miliseconds to allow before a + ## ``TimeoutError`` is raised. + new result + result.headers = newHttpHeaders() + result.userAgent = userAgent + result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = timeout + result.onProgressChanged = nil + when defined(ssl): + result.sslContext = sslContext + +type + AsyncHttpClient* = HttpClientBase[AsyncSocket] {.deprecated: [PAsyncHttpClient: AsyncHttpClient].} proc newAsyncHttpClient*(userAgent = defUserAgent, - maxRedirects = 5, sslContext = defaultSslContext): AsyncHttpClient = + maxRedirects = 5, sslContext = defaultSslContext, + proxy: Proxy = nil): AsyncHttpClient = ## Creates a new AsyncHttpClient instance. ## ## ``userAgent`` specifies the user agent that will be used when making @@ -666,29 +765,58 @@ proc newAsyncHttpClient*(userAgent = defUserAgent, ## default is 5. ## ## ``sslContext`` specifies the SSL context to use for HTTPS requests. + ## + ## ``proxy`` specifies an HTTP proxy to use for this HTTP client's + ## connections. new result - result.headers = newStringTable(modeCaseInsensitive) + result.headers = newHttpHeaders() result.userAgent = userAgent result.maxRedirects = maxRedirects + result.proxy = proxy + result.timeout = -1 # TODO + result.onProgressChanged = nil when defined(ssl): result.sslContext = sslContext -proc close*(client: AsyncHttpClient) = +proc close*(client: HttpClient | AsyncHttpClient) = ## Closes any connections held by the HTTP client. if client.connected: client.socket.close() client.connected = false -proc recvFull(socket: AsyncSocket, size: int): Future[string] {.async.} = +proc reportProgress(client: HttpClient | AsyncHttpClient, + progress: BiggestInt) {.multisync.} = + client.contentProgress += progress + client.oneSecondProgress += progress + if epochTime() - client.lastProgressReport >= 1.0: + if not client.onProgressChanged.isNil: + await client.onProgressChanged(client.contentTotal, + client.contentProgress, + client.oneSecondProgress) + client.oneSecondProgress = 0 + client.lastProgressReport = epochTime() + +proc recvFull(client: HttpClient | AsyncHttpClient, + size: int, timeout: int): Future[string] {.multisync.} = ## Ensures that all the data requested is read and returned. result = "" while true: if size == result.len: break - let data = await socket.recv(size - result.len) + + let remainingSize = size - result.len + let sizeToRecv = min(remainingSize, net.BufferSize) + + when client.socket is Socket: + let data = client.socket.recv(sizeToRecv, timeout) + else: + let data = await client.socket.recv(sizeToRecv) if data == "": break # We've been disconnected. result.add data -proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = + await reportProgress(client, data.len) + +proc parseChunks(client: HttpClient | AsyncHttpClient): Future[string] + {.multisync.} = result = "" while true: var chunkSize = 0 @@ -714,17 +842,23 @@ proc parseChunks(client: AsyncHttpClient): Future[string] {.async.} = httpError("Invalid chunk size: " & chunkSizeStr) inc(i) if chunkSize <= 0: - discard await recvFull(client.socket, 2) # Skip \c\L + discard await recvFull(client, 2, client.timeout) # Skip \c\L break - result.add await recvFull(client.socket, chunkSize) - discard await recvFull(client.socket, 2) # Skip \c\L + result.add await recvFull(client, chunkSize, client.timeout) + discard await recvFull(client, 2, client.timeout) # Skip \c\L # Trailer headers will only be sent if the request specifies that we want # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 -proc parseBody(client: AsyncHttpClient, +proc parseBody(client: HttpClient | AsyncHttpClient, headers: HttpHeaders, - httpVersion: string): Future[string] {.async.} = + httpVersion: string): Future[string] {.multisync.} = result = "" + # Reset progress from previous requests. + client.contentTotal = 0 + client.contentProgress = 0 + client.oneSecondProgress = 0 + client.lastProgressReport = 0 + if headers.getOrDefault"Transfer-Encoding" == "chunked": result = await parseChunks(client) else: @@ -733,8 +867,9 @@ proc parseBody(client: AsyncHttpClient, var contentLengthHeader = headers.getOrDefault"Content-Length" if contentLengthHeader != "": var length = contentLengthHeader.parseint() + client.contentTotal = length if length > 0: - result = await client.socket.recvFull(length) + result = await client.recvFull(length, client.timeout) if result == "": httpError("Got disconnected while trying to read body.") if result.len != length: @@ -748,12 +883,12 @@ proc parseBody(client: AsyncHttpClient, if headers.getOrDefault"Connection" == "close" or httpVersion == "1.0": var buf = "" while true: - buf = await client.socket.recvFull(4000) + buf = await client.recvFull(4000, client.timeout) if buf == "": break result.add(buf) -proc parseResponse(client: AsyncHttpClient, - getBody: bool): Future[Response] {.async.} = +proc parseResponse(client: HttpClient | AsyncHttpClient, + getBody: bool): Future[Response] {.multisync.} = var parsedStatus = false var linei = 0 var fullyRead = false @@ -761,7 +896,10 @@ proc parseResponse(client: AsyncHttpClient, result.headers = newHttpHeaders() while true: linei = 0 - line = await client.socket.recvLine() + when client is HttpClient: + line = await client.socket.recvLine(client.timeout) + else: + line = await client.socket.recvLine() if line == "": break # We've been disconnected. if line == "\c\L": fullyRead = true @@ -803,11 +941,17 @@ proc parseResponse(client: AsyncHttpClient, else: result.body = "" -proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} = +proc newConnection(client: HttpClient | AsyncHttpClient, + url: Uri) {.multisync.} = if client.currentURL.hostname != url.hostname or client.currentURL.scheme != url.scheme: if client.connected: client.close() - client.socket = newAsyncSocket() + + when client is HttpClient: + client.socket = newSocket() + elif client is AsyncHttpClient: + client.socket = newAsyncSocket() + else: {.fatal: "Unsupported client type".} # TODO: I should be able to write 'net.Port' here... let port = @@ -829,60 +973,106 @@ proc newConnection(client: AsyncHttpClient, url: Uri) {.async.} = client.currentURL = url client.connected = true -proc request*(client: AsyncHttpClient, url: string, httpMethod: string, - body = ""): Future[Response] {.async.} = +proc request*(client: HttpClient | AsyncHttpClient, url: string, + httpMethod: string, body = ""): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. ## ## Connection will kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. - ## - ## The returned future will complete once the request is completed. - let r = parseUri(url) - await newConnection(client, r) + let connectionUrl = + if client.proxy.isNil: parseUri(url) else: client.proxy.url + let requestUrl = parseUri(url) + + let savedProxy = client.proxy # client's proxy may be overwritten. + + if requestUrl.scheme == "https" and not client.proxy.isNil: + when defined(ssl): + client.proxy.url = connectionUrl + var connectUrl = requestUrl + connectUrl.scheme = "http" + connectUrl.port = "443" + let proxyResp = await request(client, $connectUrl, $HttpConnect) + + if not proxyResp.status.startsWith("200"): + raise newException(HttpRequestError, + "The proxy server rejected a CONNECT request, " & + "so a secure connection could not be established.") + client.sslContext.wrapConnectedSocket(client.socket, handshakeAsClient) + client.proxy = nil + else: + raise newException(HttpRequestError, + "SSL support not available. Cannot connect to https site over proxy.") + else: + await newConnection(client, connectionUrl) if not client.headers.hasKey("user-agent") and client.userAgent != "": client.headers["User-Agent"] = client.userAgent - var headers = generateHeaders(r, $httpMethod, client.headers, body) + var headers = generateHeaders(requestUrl, httpMethod, + client.headers, body, client.proxy) await client.socket.send(headers) if body != "": await client.socket.send(body) - result = await parseResponse(client, httpMethod != "httpHEAD") + result = await parseResponse(client, + httpMethod.toLower() notin ["head", "connect"]) -proc request*(client: AsyncHttpClient, url: string, httpMethod = httpGET, - body = ""): Future[Response] = + # Restore the clients proxy in case it was overwritten. + client.proxy = savedProxy + +proc request*(client: HttpClient | AsyncHttpClient, url: string, + httpMethod = HttpGET, body = ""): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## - ## Connection will kept alive. Further requests on the same ``client`` to + ## Connection will be kept alive. Further requests on the same ``client`` to ## the same hostname will not require a new connection to be made. The ## connection can be closed by using the ``close`` procedure. ## - ## The returned future will complete once the request is completed. - result = request(client, url, $httpMethod, body) + ## When a request is made to a different hostname, the current connection will + ## be closed. + result = await request(client, url, $httpMethod, body) -proc get*(client: AsyncHttpClient, url: string): Future[Response] {.async.} = +proc get*(client: HttpClient | AsyncHttpClient, + url: string): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a GET request. ## ## This procedure will follow redirects up to a maximum number of redirects - ## specified in ``newAsyncHttpClient``. - result = await client.request(url, httpGET) + ## specified in ``client.maxRedirects``. + result = await client.request(url, HttpGET) + + # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) - result = await client.request(redirectTo, httpGET) + result = await client.request(redirectTo, HttpGET) lastURL = redirectTo -proc post*(client: AsyncHttpClient, url: string, body = "", multipart: MultipartData = nil): Future[Response] {.async.} = +proc getContent*(client: HttpClient | AsyncHttpClient, + url: string): Future[string] {.multisync.} = + ## Connects to the hostname specified by the URL and performs a GET request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await get(client, url) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) + else: + return resp.body + +proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", + multipart: MultipartData = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a POST request. ## ## This procedure will follow redirects up to a maximum number of redirects - ## specified in ``newAsyncHttpClient``. + ## specified in ``client.maxRedirects``. let (mpHeader, mpBody) = format(multipart) template withNewLine(x): expr = @@ -895,45 +1085,29 @@ proc post*(client: AsyncHttpClient, url: string, body = "", multipart: Multipart client.headers["Content-Type"] = mpHeader.split(": ")[1] client.headers["Content-Length"] = $len(xb) - result = await client.request(url, httpPOST, xb) - -when not defined(testing) and isMainModule: - when true: - # Async - proc main() {.async.} = - var client = newAsyncHttpClient() - var resp = await client.request("http://picheta.me") - - echo("Got response: ", resp.status) - echo("Body:\n") - echo(resp.body) - - resp = await client.request("http://picheta.me/asfas.html") - echo("Got response: ", resp.status) - - resp = await client.request("http://picheta.me/aboutme.html") - echo("Got response: ", resp.status) - - resp = await client.request("http://nim-lang.org/") - echo("Got response: ", resp.status) - - resp = await client.request("http://nim-lang.org/download.html") - echo("Got response: ", resp.status) - - waitFor main() + result = await client.request(url, HttpPOST, xb) + # Handle redirects. + var lastURL = url + for i in 1..client.maxRedirects: + if result.status.redirection(): + let redirectTo = getNewLocation(lastURL, result.headers) + var meth = if result.status != "307": HttpGet else: HttpPost + result = await client.request(redirectTo, meth, xb) + lastURL = redirectTo +proc postContent*(client: HttpClient | AsyncHttpClient, url: string, + body = "", + multipart: MultipartData = nil): Future[string] + {.multisync.} = + ## Connects to the hostname specified by the URL and performs a POST request. + ## + ## This procedure will follow redirects up to a maximum number of redirects + ## specified in ``client.maxRedirects``. + ## + ## A ``HttpRequestError`` will be raised if the server responds with a + ## client error (status code 4xx) or a server error (status code 5xx). + let resp = await post(client, url, body, multipart) + if resp.code.is4xx or resp.code.is5xx: + raise newException(HttpRequestError, resp.status) else: - #downloadFile("http://force7.de/nim/index.html", "nimindex.html") - #downloadFile("http://www.httpwatch.com/", "ChunkTest.html") - #downloadFile("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com", - # "validator.html") - - #var r = get("http://validator.w3.org/check?uri=http%3A%2F%2Fgoogle.com& - # charset=%28detect+automatically%29&doctype=Inline&group=0") - - var data = newMultipartData() - data["output"] = "soap12" - data["uploaded_file"] = ("test.html", "text/html", - "<html><head></head><body><p>test</p></body></html>") - - echo postContent("http://validator.w3.org/check", multipart=data) + return resp.body diff --git a/lib/pure/httpcore.nim b/lib/pure/httpcore.nim index 562d16c19..8147f1c50 100644 --- a/lib/pure/httpcore.nim +++ b/lib/pure/httpcore.nim @@ -18,59 +18,87 @@ type HttpHeaderValues* = distinct seq[string] - HttpCode* = enum - Http100 = "100 Continue", - Http101 = "101 Switching Protocols", - Http200 = "200 OK", - Http201 = "201 Created", - Http202 = "202 Accepted", - Http203 = "203 Non-Authoritative Information", - Http204 = "204 No Content", - Http205 = "205 Reset Content", - Http206 = "206 Partial Content", - Http300 = "300 Multiple Choices", - Http301 = "301 Moved Permanently", - Http302 = "302 Found", - Http303 = "303 See Other", - Http304 = "304 Not Modified", - Http305 = "305 Use Proxy", - Http307 = "307 Temporary Redirect", - Http400 = "400 Bad Request", - Http401 = "401 Unauthorized", - Http403 = "403 Forbidden", - Http404 = "404 Not Found", - Http405 = "405 Method Not Allowed", - Http406 = "406 Not Acceptable", - Http407 = "407 Proxy Authentication Required", - Http408 = "408 Request Timeout", - Http409 = "409 Conflict", - Http410 = "410 Gone", - Http411 = "411 Length Required", - Http412 = "412 Precondition Failed", - Http413 = "413 Request Entity Too Large", - Http414 = "414 Request-URI Too Long", - Http415 = "415 Unsupported Media Type", - Http416 = "416 Requested Range Not Satisfiable", - Http417 = "417 Expectation Failed", - Http418 = "418 I'm a teapot", - Http421 = "421 Misdirected Request", - Http422 = "422 Unprocessable Entity", - Http426 = "426 Upgrade Required", - Http428 = "428 Precondition Required", - Http429 = "429 Too Many Requests", - Http431 = "431 Request Header Fields Too Large", - Http451 = "451 Unavailable For Legal Reasons", - Http500 = "500 Internal Server Error", - Http501 = "501 Not Implemented", - Http502 = "502 Bad Gateway", - Http503 = "503 Service Unavailable", - Http504 = "504 Gateway Timeout", - Http505 = "505 HTTP Version Not Supported" + # The range starts at '0' so that we don't have to explicitly initialise + # it. See: http://irclogs.nim-lang.org/19-09-2016.html#19:48:27 for context. + HttpCode* = distinct range[0 .. 599] HttpVersion* = enum HttpVer11, HttpVer10 + HttpMethod* = enum ## the requested HttpMethod + HttpHead, ## Asks for the response identical to the one that would + ## correspond to a GET request, but without the response + ## body. + HttpGet, ## Retrieves the specified resource. + HttpPost, ## Submits data to be processed to the identified + ## resource. The data is included in the body of the + ## request. + HttpPut, ## Uploads a representation of the specified resource. + HttpDelete, ## Deletes the specified resource. + HttpTrace, ## Echoes back the received request, so that a client + ## can see what intermediate servers are adding or + ## changing in the request. + HttpOptions, ## Returns the HTTP methods that the server supports + ## for specified address. + HttpConnect, ## Converts the request connection to a transparent + ## TCP/IP tunnel, usually used for proxies. + HttpPatch ## Applies partial modifications to a resource. + +{.deprecated: [httpGet: HttpGet, httpHead: HttpHead, httpPost: HttpPost, + httpPut: HttpPut, httpDelete: HttpDelete, httpTrace: HttpTrace, + httpOptions: HttpOptions, httpConnect: HttpConnect].} + + +const + Http100* = HttpCode(100) + Http101* = HttpCode(101) + Http200* = HttpCode(200) + Http201* = HttpCode(201) + Http202* = HttpCode(202) + Http203* = HttpCode(203) + Http204* = HttpCode(204) + Http205* = HttpCode(205) + Http206* = HttpCode(206) + Http300* = HttpCode(300) + Http301* = HttpCode(301) + Http302* = HttpCode(302) + Http303* = HttpCode(303) + Http304* = HttpCode(304) + Http305* = HttpCode(305) + Http307* = HttpCode(307) + Http400* = HttpCode(400) + Http401* = HttpCode(401) + Http403* = HttpCode(403) + Http404* = HttpCode(404) + Http405* = HttpCode(405) + Http406* = HttpCode(406) + Http407* = HttpCode(407) + Http408* = HttpCode(408) + Http409* = HttpCode(409) + Http410* = HttpCode(410) + Http411* = HttpCode(411) + Http412* = HttpCode(412) + Http413* = HttpCode(413) + Http414* = HttpCode(414) + Http415* = HttpCode(415) + Http416* = HttpCode(416) + Http417* = HttpCode(417) + Http418* = HttpCode(418) + Http421* = HttpCode(421) + Http422* = HttpCode(422) + Http426* = HttpCode(426) + Http428* = HttpCode(428) + Http429* = HttpCode(429) + Http431* = HttpCode(431) + Http451* = HttpCode(451) + Http500* = HttpCode(500) + Http501* = HttpCode(501) + Http502* = HttpCode(502) + Http503* = HttpCode(503) + Http504* = HttpCode(504) + Http505* = HttpCode(505) + const headerLimit* = 10_000 proc newHttpHeaders*(): HttpHeaders = @@ -81,7 +109,7 @@ proc newHttpHeaders*(keyValuePairs: openarray[tuple[key: string, val: string]]): HttpHeaders = var pairs: seq[tuple[key: string, val: seq[string]]] = @[] for pair in keyValuePairs: - pairs.add((pair.key.toLower(), @[pair.val])) + pairs.add((pair.key.toLowerAscii(), @[pair.val])) new result result.table = newTable[string, seq[string]](pairs) @@ -96,7 +124,7 @@ proc `[]`*(headers: HttpHeaders, key: string): HttpHeaderValues = ## ## To access multiple values of a key, use the overloaded ``[]`` below or ## to get all of them access the ``table`` field directly. - return headers.table[key.toLower].HttpHeaderValues + return headers.table[key.toLowerAscii].HttpHeaderValues converter toString*(values: HttpHeaderValues): string = return seq[string](values)[0] @@ -105,26 +133,26 @@ proc `[]`*(headers: HttpHeaders, key: string, i: int): string = ## Returns the ``i``'th value associated with the given key. If there are ## no values associated with the key or the ``i``'th value doesn't exist, ## an exception is raised. - return headers.table[key.toLower][i] + return headers.table[key.toLowerAscii][i] proc `[]=`*(headers: HttpHeaders, key, value: string) = ## Sets the header entries associated with ``key`` to the specified value. ## Replaces any existing values. - headers.table[key.toLower] = @[value] + headers.table[key.toLowerAscii] = @[value] proc `[]=`*(headers: HttpHeaders, key: string, value: seq[string]) = ## Sets the header entries associated with ``key`` to the specified list of ## values. ## Replaces any existing values. - headers.table[key.toLower] = value + headers.table[key.toLowerAscii] = value proc add*(headers: HttpHeaders, key, value: string) = ## Adds the specified value to the specified key. Appends to any existing ## values associated with the key. - if not headers.table.hasKey(key.toLower): - headers.table[key.toLower] = @[value] + if not headers.table.hasKey(key.toLowerAscii): + headers.table[key.toLowerAscii] = @[value] else: - headers.table[key.toLower].add(value) + headers.table[key.toLowerAscii].add(value) iterator pairs*(headers: HttpHeaders): tuple[key, value: string] = ## Yields each key, value pair. @@ -136,10 +164,10 @@ proc contains*(values: HttpHeaderValues, value: string): bool = ## Determines if ``value`` is one of the values inside ``values``. Comparison ## is performed without case sensitivity. for val in seq[string](values): - if val.toLower == value.toLower: return true + if val.toLowerAscii == value.toLowerAscii: return true proc hasKey*(headers: HttpHeaders, key: string): bool = - return headers.table.hasKey(key.toLower()) + return headers.table.hasKey(key.toLowerAscii()) proc getOrDefault*(headers: HttpHeaders, key: string, default = @[""].HttpHeaderValues): HttpHeaderValues = @@ -188,6 +216,90 @@ proc `==`*(protocol: tuple[orig: string, major, minor: int], of HttpVer10: 0 result = protocol.major == major and protocol.minor == minor +proc contains*(methods: set[HttpMethod], x: string): bool = + return parseEnum[HttpMethod](x) in methods + +proc `$`*(code: HttpCode): string = + ## Converts the specified ``HttpCode`` into a HTTP status. + ## + ## For example: + ## + ## .. code-block:: nim + ## doAssert($Http404 == "404 Not Found") + case code.int + of 100: "100 Continue" + of 101: "101 Switching Protocols" + of 200: "200 OK" + of 201: "201 Created" + of 202: "202 Accepted" + of 203: "203 Non-Authoritative Information" + of 204: "204 No Content" + of 205: "205 Reset Content" + of 206: "206 Partial Content" + of 300: "300 Multiple Choices" + of 301: "301 Moved Permanently" + of 302: "302 Found" + of 303: "303 See Other" + of 304: "304 Not Modified" + of 305: "305 Use Proxy" + of 307: "307 Temporary Redirect" + of 400: "400 Bad Request" + of 401: "401 Unauthorized" + of 403: "403 Forbidden" + of 404: "404 Not Found" + of 405: "405 Method Not Allowed" + of 406: "406 Not Acceptable" + of 407: "407 Proxy Authentication Required" + of 408: "408 Request Timeout" + of 409: "409 Conflict" + of 410: "410 Gone" + of 411: "411 Length Required" + of 412: "412 Precondition Failed" + of 413: "413 Request Entity Too Large" + of 414: "414 Request-URI Too Long" + of 415: "415 Unsupported Media Type" + of 416: "416 Requested Range Not Satisfiable" + of 417: "417 Expectation Failed" + of 418: "418 I'm a teapot" + of 421: "421 Misdirected Request" + of 422: "422 Unprocessable Entity" + of 426: "426 Upgrade Required" + of 428: "428 Precondition Required" + of 429: "429 Too Many Requests" + of 431: "431 Request Header Fields Too Large" + of 451: "451 Unavailable For Legal Reasons" + of 500: "500 Internal Server Error" + of 501: "501 Not Implemented" + of 502: "502 Bad Gateway" + of 503: "503 Service Unavailable" + of 504: "504 Gateway Timeout" + of 505: "505 HTTP Version Not Supported" + else: $(int(code)) + +proc `==`*(a, b: HttpCode): bool {.borrow.} + +proc `==`*(rawCode: string, code: HttpCode): bool = + return rawCode.toLower() == ($code).toLower() + +proc is2xx*(code: HttpCode): bool = + ## Determines whether ``code`` is a 2xx HTTP status code. + return code.int in {200 .. 299} + +proc is3xx*(code: HttpCode): bool = + ## Determines whether ``code`` is a 3xx HTTP status code. + return code.int in {300 .. 399} + +proc is4xx*(code: HttpCode): bool = + ## Determines whether ``code`` is a 4xx HTTP status code. + return code.int in {400 .. 499} + +proc is5xx*(code: HttpCode): bool = + ## Determines whether ``code`` is a 5xx HTTP status code. + return code.int in {500 .. 599} + +proc `$`*(httpMethod: HttpMethod): string = + return (system.`$`(httpMethod))[4 .. ^1].toUpper() + when isMainModule: var test = newHttpHeaders() test["Connection"] = @["Upgrade", "Close"] diff --git a/lib/pure/includes/asyncfutures.nim b/lib/pure/includes/asyncfutures.nim new file mode 100644 index 000000000..d78464a91 --- /dev/null +++ b/lib/pure/includes/asyncfutures.nim @@ -0,0 +1,295 @@ + +# TODO: This shouldn't need to be included, but should ideally be exported. +type + FutureBase* = ref object of RootObj ## Untyped future. + cb: proc () {.closure,gcsafe.} + finished: bool + error*: ref Exception ## Stored exception + errorStackTrace*: string + when not defined(release): + stackTrace: string ## For debugging purposes only. + id: int + fromProc: string + + Future*[T] = ref object of FutureBase ## Typed future. + value: T ## Stored value + + FutureVar*[T] = distinct Future[T] + + FutureError* = object of Exception + cause*: FutureBase + +{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} + +when not defined(release): + var currentID = 0 + +proc callSoon*(cbproc: proc ()) {.gcsafe.} + +proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = + ## Creates a new future. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + new(result) + result.finished = false + when not defined(release): + result.stackTrace = getStackTrace() + result.id = currentID + result.fromProc = fromProc + currentID.inc() + +proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = + ## Create a new ``FutureVar``. This Future type is ideally suited for + ## situations where you want to avoid unnecessary allocations of Futures. + ## + ## Specifying ``fromProc``, which is a string specifying the name of the proc + ## that this future belongs to, is a good habit as it helps with debugging. + result = FutureVar[T](newFuture[T](fromProc)) + +proc clean*[T](future: FutureVar[T]) = + ## Resets the ``finished`` status of ``future``. + Future[T](future).finished = false + Future[T](future).error = nil + +proc checkFinished[T](future: Future[T]) = + ## Checks whether `future` is finished. If it is then raises a + ## ``FutureError``. + when not defined(release): + if future.finished: + var msg = "" + msg.add("An attempt was made to complete a Future more than once. ") + msg.add("Details:") + msg.add("\n Future ID: " & $future.id) + msg.add("\n Created in proc: " & future.fromProc) + msg.add("\n Stack trace to moment of creation:") + msg.add("\n" & indent(future.stackTrace.strip(), 4)) + when T is string: + msg.add("\n Contents (string): ") + msg.add("\n" & indent(future.value.repr, 4)) + msg.add("\n Stack trace to moment of secondary completion:") + msg.add("\n" & indent(getStackTrace().strip(), 4)) + var err = newException(FutureError, msg) + err.cause = future + raise err + +proc complete*[T](future: Future[T], val: T) = + ## Completes ``future`` with value ``val``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.value = val + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*(future: Future[void]) = + ## Completes a void ``future``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + assert(future.error == nil) + future.finished = true + if future.cb != nil: + future.cb() + +proc complete*[T](future: FutureVar[T]) = + ## Completes a ``FutureVar``. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + if fut.cb != nil: + fut.cb() + +proc complete*[T](future: FutureVar[T], val: T) = + ## Completes a ``FutureVar`` with value ``val``. + ## + ## Any previously stored value will be overwritten. + template fut: expr = Future[T](future) + checkFinished(fut) + assert(fut.error == nil) + fut.finished = true + fut.value = val + if fut.cb != nil: + fut.cb() + +proc fail*[T](future: Future[T], error: ref Exception) = + ## Completes ``future`` with ``error``. + #assert(not future.finished, "Future already finished, cannot finish twice.") + checkFinished(future) + future.finished = true + future.error = error + future.errorStackTrace = + if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) + if future.cb != nil: + future.cb() + else: + # This is to prevent exceptions from being silently ignored when a future + # is discarded. + # TODO: This may turn out to be a bad idea. + # Turns out this is a bad idea. + #raise error + discard + +proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + ## + ## **Note**: You most likely want the other ``callback`` setter which + ## passes ``future`` as a param to the callback. + future.cb = cb + if future.finished: + callSoon(future.cb) + +proc `callback=`*[T](future: Future[T], + cb: proc (future: Future[T]) {.closure,gcsafe.}) = + ## Sets the callback proc to be called when the future completes. + ## + ## If future has already completed then ``cb`` will be called immediately. + future.callback = proc () = cb(future) + +proc injectStacktrace[T](future: Future[T]) = + # TODO: Come up with something better. + when not defined(release): + var msg = "" + msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") + + if not future.errorStackTrace.isNil and future.errorStackTrace != "": + msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) + else: + msg.add("\n Empty or nil stack trace.") + future.error.msg.add(msg) + +proc read*[T](future: Future[T] | FutureVar[T]): T = + ## Retrieves the value of ``future``. Future must be finished otherwise + ## this function will fail with a ``ValueError`` exception. + ## + ## If the result of the future is an error then that error will be raised. + {.push hint[ConvFromXtoItselfNotNeeded]: off.} + let fut = Future[T](future) + {.pop.} + if fut.finished: + if fut.error != nil: + injectStacktrace(fut) + raise fut.error + when T isnot void: + return fut.value + else: + # TODO: Make a custom exception type for this? + raise newException(ValueError, "Future still in progress.") + +proc readError*[T](future: Future[T]): ref Exception = + ## Retrieves the exception stored in ``future``. + ## + ## An ``ValueError`` exception will be thrown if no exception exists + ## in the specified Future. + if future.error != nil: return future.error + else: + raise newException(ValueError, "No error in future.") + +proc mget*[T](future: FutureVar[T]): var T = + ## Returns a mutable value stored in ``future``. + ## + ## Unlike ``read``, this function will not raise an exception if the + ## Future has not been finished. + result = Future[T](future).value + +proc finished*[T](future: Future[T] | FutureVar[T]): bool = + ## Determines whether ``future`` has completed. + ## + ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. + (Future[T](future)).finished + +proc failed*(future: FutureBase): bool = + ## Determines whether ``future`` completed with an error. + return future.error != nil + +proc asyncCheck*[T](future: Future[T]) = + ## Sets a callback on ``future`` which raises an exception if the future + ## finished with an error. + ## + ## This should be used instead of ``discard`` to discard void futures. + future.callback = + proc () = + if future.failed: + injectStacktrace(future) + raise future.error + +proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once both ``fut1`` and ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`and`") + fut1.callback = + proc () = + if not retFuture.finished: + if fut1.failed: retFuture.fail(fut1.error) + elif fut2.finished: retFuture.complete() + fut2.callback = + proc () = + if not retFuture.finished: + if fut2.failed: retFuture.fail(fut2.error) + elif fut1.finished: retFuture.complete() + return retFuture + +proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = + ## Returns a future which will complete once either ``fut1`` or ``fut2`` + ## complete. + var retFuture = newFuture[void]("asyncdispatch.`or`") + proc cb[X](fut: Future[X]) = + if fut.failed: retFuture.fail(fut.error) + if not retFuture.finished: retFuture.complete() + fut1.callback = cb[T] + fut2.callback = cb[Y] + return retFuture + +proc all*[T](futs: varargs[Future[T]]): auto = + ## Returns a future which will complete once + ## all futures in ``futs`` complete. + ## + ## If the awaited futures are not ``Future[void]``, the returned future + ## will hold the values of all awaited futures in a sequence. + ## + ## If the awaited futures *are* ``Future[void]``, + ## this proc returns ``Future[void]``. + + when T is void: + var + retFuture = newFuture[void]("asyncdispatch.all") + completedFutures = 0 + + let totalFutures = len(futs) + + for fut in futs: + fut.callback = proc(f: Future[T]) = + if f.failed: + retFuture.fail(f.error) + elif not retFuture.finished: + inc(completedFutures) + + if completedFutures == totalFutures: + retFuture.complete() + + return retFuture + + else: + var + retFuture = newFuture[seq[T]]("asyncdispatch.all") + retValues = newSeq[T](len(futs)) + completedFutures = 0 + + for i, fut in futs: + proc setCallback(i: int) = + fut.callback = proc(f: Future[T]) = + if f.failed: + retFuture.fail(f.error) + elif not retFuture.finished: + retValues[i] = f.read() + inc(completedFutures) + + if completedFutures == len(retValues): + retFuture.complete(retValues) + + setCallback(i) + + return retFuture diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index a5d5d2c01..adb3497ac 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -44,14 +44,21 @@ when defined(nimdoc): Event* {.pure.} = enum ## An enum which hold event types - Read, ## Descriptor is available for read - Write, ## Descriptor is available for write - Timer, ## Timer descriptor is completed - Signal, ## Signal is raised - Process, ## Process is finished - Vnode, ## Currently not supported - User, ## User event is raised - Error ## Error happens while waiting, for descriptor + Read, ## Descriptor is available for read + Write, ## Descriptor is available for write + Timer, ## Timer descriptor is completed + Signal, ## Signal is raised + Process, ## Process is finished + Vnode, ## BSD specific file change happens + User, ## User event is raised + Error, ## Error happens while waiting, for descriptor + VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occured) + VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occured) + VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) + VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed) + VnodeLink, ## NOTE_LINK (BSD specific, file link count changed) + VnodeRename, ## NOTE_RENAME (BSD specific, file renamed) + VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occured) ReadyKey*[T] = object ## An object which holds result for descriptor @@ -107,6 +114,15 @@ when defined(nimdoc): ## ``data`` application-defined data, which to be passed, when ## ``ev`` happens. + proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], + data: T) = + ## Registers selector BSD/MacOSX specific vnode events for file + ## descriptor ``fd`` and events ``events``. + ## ``data`` application-defined data, which to be passed, when + ## vnode event happens. + ## + ## This function is supported only by BSD and MacOSX. + proc newSelectEvent*(): SelectEvent = ## Creates new event ``SelectEvent``. @@ -194,7 +210,9 @@ else: deallocShared(cast[pointer](sa)) type Event* {.pure.} = enum - Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot + Read, Write, Timer, Signal, Process, Vnode, User, Error, Oneshot, + VnodeWrite, VnodeDelete, VnodeExtend, VnodeAttrib, VnodeLink, + VnodeRename, VnodeRevoke ReadyKey*[T] = object fd* : int diff --git a/lib/pure/ioselects/ioselectors_epoll.nim b/lib/pure/ioselects/ioselectors_epoll.nim index 92b2cdc07..4bc6e9d51 100644 --- a/lib/pure/ioselects/ioselectors_epoll.nim +++ b/lib/pure/ioselects/ioselectors_epoll.nim @@ -407,8 +407,8 @@ proc selectInto*[T](s: Selector[T], timeout: int, inc(i) continue elif Event.User in skey.events: - var data: uint = 0 - if posix.read(fdi.cint, addr data, sizeof(uint)) != sizeof(uint): + var data: uint64 = 0 + if posix.read(fdi.cint, addr data, sizeof(uint64)) != sizeof(uint64): let err = osLastError() if err == OSErrorCode(EAGAIN): inc(i) diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index 3e86f19aa..cdaeeae26 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -262,6 +262,30 @@ proc registerEvent*[T](s: Selector[T], ev: SelectEvent, data: T) = modifyKQueue(s, fdi.uint, EVFILT_READ, EV_ADD, 0, 0, nil) inc(s.count) +template processVnodeEvents(events: set[Event]): cuint = + var rfflags = 0.cuint + if events == {Event.VnodeWrite, Event.VnodeDelete, Event.VnodeExtend, + Event.VnodeAttrib, Event.VnodeLink, Event.VnodeRename, + Event.VnodeRevoke}: + rfflags = NOTE_DELETE or NOTE_WRITE or NOTE_EXTEND or NOTE_ATTRIB or + NOTE_LINK or NOTE_RENAME or NOTE_REVOKE + else: + if Event.VnodeDelete in events: rfflags = rfflags or NOTE_DELETE + if Event.VnodeWrite in events: rfflags = rfflags or NOTE_WRITE + if Event.VnodeExtend in events: rfflags = rfflags or NOTE_EXTEND + if Event.VnodeAttrib in events: rfflags = rfflags or NOTE_ATTRIB + if Event.VnodeLink in events: rfflags = rfflags or NOTE_LINK + if Event.VnodeRename in events: rfflags = rfflags or NOTE_RENAME + if Event.VnodeRevoke in events: rfflags = rfflags or NOTE_REVOKE + rfflags + +proc registerVnode*[T](s: Selector[T], fd: cint, events: set[Event], data: T) = + let fdi = fd.int + setKey(s, fdi, fdi, {Event.Vnode} + events, 0, data) + var fflags = processVnodeEvents(events) + modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_ADD or EV_CLEAR, fflags, 0, nil) + inc(s.count) + proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = let fdi = int(fd) s.checkFd(fdi) @@ -295,6 +319,9 @@ proc unregister*[T](s: Selector[T], fd: int|SocketHandle) = discard posix.close(cint(pkey.key.fd)) modifyKQueue(s, fdi.uint, EVFILT_PROC, EV_DELETE, 0, 0, nil) dec(s.count) + elif Event.Vnode in pkey.events: + modifyKQueue(s, fdi.uint, EVFILT_VNODE, EV_DELETE, 0, 0, nil) + dec(s.count) elif Event.User in pkey.events: modifyKQueue(s, fdi.uint, EVFILT_READ, EV_DELETE, 0, 0, nil) dec(s.count) @@ -392,6 +419,20 @@ proc selectInto*[T](s: Selector[T], timeout: int, of EVFILT_VNODE: pkey = addr(s.fds[kevent.ident.int]) pkey.key.events = {Event.Vnode} + if (kevent.fflags and NOTE_DELETE) != 0: + pkey.key.events.incl(Event.VnodeDelete) + if (kevent.fflags and NOTE_WRITE) != 0: + pkey.key.events.incl(Event.VnodeWrite) + if (kevent.fflags and NOTE_EXTEND) != 0: + pkey.key.events.incl(Event.VnodeExtend) + if (kevent.fflags and NOTE_ATTRIB) != 0: + pkey.key.events.incl(Event.VnodeAttrib) + if (kevent.fflags and NOTE_LINK) != 0: + pkey.key.events.incl(Event.VnodeLink) + if (kevent.fflags and NOTE_RENAME) != 0: + pkey.key.events.incl(Event.VnodeRename) + if (kevent.fflags and NOTE_REVOKE) != 0: + pkey.key.events.incl(Event.VnodeRevoke) of EVFILT_SIGNAL: pkey = addr(s.fds[cast[int](kevent.udata)]) pkey.key.events = {Event.Signal} diff --git a/lib/pure/ioselects/ioselectors_poll.nim b/lib/pure/ioselects/ioselectors_poll.nim index d2a0a1273..56be35c70 100644 --- a/lib/pure/ioselects/ioselectors_poll.nim +++ b/lib/pure/ioselects/ioselectors_poll.nim @@ -245,7 +245,7 @@ proc selectInto*[T](s: Selector[T], timeout: int, skey.key.events.incl(Event.Read) if Event.User in skey.events: var data: uint64 = 0 - if posix.read(fd, addr data, sizeof(int)) != sizeof(int): + if posix.read(fd, addr data, sizeof(uint64)) != sizeof(uint64): let err = osLastError() if err != OSErrorCode(EAGAIN): raiseOSError(osLastError()) diff --git a/lib/pure/ioselects/ioselectors_select.nim b/lib/pure/ioselects/ioselectors_select.nim index f8099f9a0..ddb70b507 100644 --- a/lib/pure/ioselects/ioselectors_select.nim +++ b/lib/pure/ioselects/ioselectors_select.nim @@ -155,9 +155,9 @@ when defined(windows): result.wsock = wsock proc setEvent*(ev: SelectEvent) = - var data: int = 1 + var data: uint64 = 1 if winlean.send(ev.wsock, cast[pointer](addr data), - cint(sizeof(int)), 0) != sizeof(int): + cint(sizeof(uint64)), 0) != sizeof(uint64): raiseOSError(osLastError()) proc close*(ev: SelectEvent) = diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 19947fbc2..0b7908c02 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -24,6 +24,8 @@ ## jobj["test"] = newJFloat(0.7) # create or update ## echo($jobj["test"].fnum) ## echo($jobj["key2"].bval) +## echo jobj{"missing key"}.getFNum(0.1) # read a float value using a default +## jobj{"a", "b", "c"} = newJFloat(3.3) # created nested keys ## ## Results in: ## @@ -580,7 +582,7 @@ type of JNull: nil of JObject: - fields*: Table[string, JsonNode] + fields*: OrderedTable[string, JsonNode] of JArray: elems*: seq[JsonNode] @@ -630,7 +632,7 @@ proc newJObject*(): JsonNode = ## Creates a new `JObject JsonNode` new(result) result.kind = JObject - result.fields = initTable[string, JsonNode](4) + result.fields = initOrderedTable[string, JsonNode](4) proc newJArray*(): JsonNode = ## Creates a new `JArray JsonNode` @@ -670,8 +672,8 @@ proc getBVal*(n: JsonNode, default: bool = false): bool = else: return n.bval proc getFields*(n: JsonNode, - default = initTable[string, JsonNode](4)): - Table[string, JsonNode] = + default = initOrderedTable[string, JsonNode](4)): + OrderedTable[string, JsonNode] = ## Retrieves the key, value pairs of a `JObject JsonNode`. ## ## Returns ``default`` if ``n`` is not a ``JObject``, or if ``n`` is nil. @@ -760,12 +762,12 @@ proc toJson(x: NimNode): NimNode {.compiletime.} = result = prefix(result, "%") -macro `%*`*(x: expr): expr = +macro `%*`*(x: untyped): untyped = ## Convert an expression to a JsonNode directly, without having to specify ## `%` for every element. result = toJson(x) -proc `==`* (a,b: JsonNode): bool = +proc `==`* (a, b: JsonNode): bool = ## Check two nodes for equality if a.isNil: if b.isNil: return true @@ -773,23 +775,29 @@ proc `==`* (a,b: JsonNode): bool = elif b.isNil or a.kind != b.kind: return false else: - return case a.kind + case a.kind of JString: - a.str == b.str + result = a.str == b.str of JInt: - a.num == b.num + result = a.num == b.num of JFloat: - a.fnum == b.fnum + result = a.fnum == b.fnum of JBool: - a.bval == b.bval + result = a.bval == b.bval of JNull: - true + result = true of JArray: - a.elems == b.elems + result = a.elems == b.elems of JObject: - a.fields == b.fields + # we cannot use OrderedTable's equality here as + # the order does not matter for equality here. + if a.fields.len != b.fields.len: return false + for key, val in a.fields: + if not b.fields.hasKey(key): return false + if b.fields[key] != val: return false + result = true -proc hash*(n: Table[string, JsonNode]): Hash {.noSideEffect.} +proc hash*(n: OrderedTable[string, JsonNode]): Hash {.noSideEffect.} proc hash*(n: JsonNode): Hash = ## Compute the hash for a JSON node @@ -807,9 +815,9 @@ proc hash*(n: JsonNode): Hash = of JString: result = hash(n.str) of JNull: - result = hash(0) + result = Hash(0) -proc hash*(n: Table[string, JsonNode]): Hash = +proc hash*(n: OrderedTable[string, JsonNode]): Hash = for key, val in n: result = result xor (hash(key) !& hash(val)) result = !$result @@ -1195,19 +1203,19 @@ else: proc len(x: JSObject): int = assert x.getVarType == JArray asm """ - return `x`.length; + `result` = `x`.length; """ proc `[]`(x: JSObject, y: string): JSObject = assert x.getVarType == JObject asm """ - return `x`[`y`]; + `result` = `x`[`y`]; """ proc `[]`(x: JSObject, y: int): JSObject = assert x.getVarType == JArray asm """ - return `x`[`y`]; + `result` = `x`[`y`]; """ proc convertObject(x: JSObject): JsonNode = diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index 6a27e6af8..b23b1e5bb 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -47,7 +47,9 @@ ## **Warning:** The global list of handlers is a thread var, this means that ## the handlers must be re-added in each thread. -import strutils, os, times +import strutils, times +when not defined(js): + import os type Level* = enum ## logging level @@ -77,21 +79,24 @@ type ConsoleLogger* = ref object of Logger ## logger that writes the messages to the ## console - FileLogger* = ref object of Logger ## logger that writes the messages to a file - file*: File ## the wrapped file. +when not defined(js): + type + FileLogger* = ref object of Logger ## logger that writes the messages to a file + file*: File ## the wrapped file. - RollingFileLogger* = ref object of FileLogger ## logger that writes the - ## messages to a file and - ## performs log rotation - maxLines: int # maximum number of lines - curLine : int - baseName: string # initial filename - baseMode: FileMode # initial file mode - logFiles: int # how many log files already created, e.g. basename.1, basename.2... - bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) + RollingFileLogger* = ref object of FileLogger ## logger that writes the + ## messages to a file and + ## performs log rotation + maxLines: int # maximum number of lines + curLine : int + baseName: string # initial filename + baseMode: FileMode # initial file mode + logFiles: int # how many log files already created, e.g. basename.1, basename.2... + bufSize: int # size of output buffer (-1: use system defaults, 0: unbuffered, >0: fixed buffer size) -{.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger, - PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} + {.deprecated: [PFileLogger: FileLogger, PRollingFileLogger: RollingFileLogger].} + +{.deprecated: [TLevel: Level, PLogger: Logger, PConsoleLogger: ConsoleLogger].} var level {.threadvar.}: Level ## global log filter @@ -112,7 +117,7 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str else: inc(i) var v = "" - var app = getAppFilename() + let app = when defined(js): "" else: getAppFilename() while frmt[i] in IdentChars: v.add(toLower(frmt[i])) inc(i) @@ -121,8 +126,10 @@ proc substituteLog*(frmt: string, level: Level, args: varargs[string, `$`]): str of "time": result.add(getClockStr()) of "datetime": result.add(getDateStr() & "T" & getClockStr()) of "app": result.add(app) - of "appdir": result.add(app.splitFile.dir) - of "appname": result.add(app.splitFile.name) + of "appdir": + when not defined(js): result.add(app.splitFile.dir) + of "appname": + when not defined(js): result.add(app.splitFile.name) of "levelid": result.add(LevelNames[level][0]) of "levelname": result.add(LevelNames[level]) else: discard @@ -139,19 +146,13 @@ method log*(logger: Logger, level: Level, args: varargs[string, `$`]) {. method log*(logger: ConsoleLogger, level: Level, args: varargs[string, `$`]) = ## Logs to the console using ``logger`` only. if level >= logging.level and level >= logger.levelThreshold: - writeLine(stdout, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(stdout) - -method log*(logger: FileLogger, level: Level, args: varargs[string, `$`]) = - ## Logs to a file using ``logger`` only. - if level >= logging.level and level >= logger.levelThreshold: - writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) - -proc defaultFilename*(): string = - ## Returns the default filename for a logger. - var (path, name, _) = splitFile(getAppFilename()) - result = changeFileExt(path / name, "log") + let ln = substituteLog(logger.fmtStr, level, args) + when defined(js): + let cln: cstring = ln + {.emit: "console.log(`cln`);".} + else: + writeLine(stdout, ln) + if level in {lvlError, lvlFatal}: flushFile(stdout) proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr): ConsoleLogger = ## Creates a new console logger. This logger logs to the console. @@ -159,87 +160,99 @@ proc newConsoleLogger*(levelThreshold = lvlAll, fmtStr = defaultFmtStr): Console result.fmtStr = fmtStr result.levelThreshold = levelThreshold -proc newFileLogger*(filename = defaultFilename(), - mode: FileMode = fmAppend, - levelThreshold = lvlAll, - fmtStr = defaultFmtStr, - bufSize: int = -1): FileLogger = - ## Creates a new file logger. This logger logs to a file. - ## Use ``bufSize`` as size of the output buffer when writing the file - ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). - new(result) - result.levelThreshold = levelThreshold - result.file = open(filename, mode, bufSize = bufSize) - result.fmtStr = fmtStr - -# ------ - -proc countLogLines(logger: RollingFileLogger): int = - result = 0 - for line in logger.file.lines(): - result.inc() - -proc countFiles(filename: string): int = - # Example: file.log.1 - result = 0 - let (dir, name, ext) = splitFile(filename) - for kind, path in walkDir(dir): - if kind == pcFile: - let llfn = name & ext & ExtSep - if path.extractFilename.startsWith(llfn): - let numS = path.extractFilename[llfn.len .. ^1] - try: - let num = parseInt(numS) - if num > result: - result = num - except ValueError: discard - -proc newRollingFileLogger*(filename = defaultFilename(), - mode: FileMode = fmReadWrite, - levelThreshold = lvlAll, - fmtStr = defaultFmtStr, - maxLines = 1000, - bufSize: int = -1): RollingFileLogger = - ## Creates a new rolling file logger. Once a file reaches ``maxLines`` lines - ## a new log file will be started and the old will be renamed. - ## Use ``bufSize`` as size of the output buffer when writing the file - ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). - new(result) - result.levelThreshold = levelThreshold - result.fmtStr = fmtStr - result.maxLines = maxLines - result.bufSize = bufSize - result.file = open(filename, mode, bufSize=result.bufSize) - result.curLine = 0 - result.baseName = filename - result.baseMode = mode - - result.logFiles = countFiles(filename) - - if mode == fmAppend: - # We need to get a line count because we will be appending to the file. - result.curLine = countLogLines(result) - -proc rotate(logger: RollingFileLogger) = - let (dir, name, ext) = splitFile(logger.baseName) - for i in countdown(logger.logFiles, 0): - let srcSuff = if i != 0: ExtSep & $i else: "" - moveFile(dir / (name & ext & srcSuff), - dir / (name & ext & ExtSep & $(i+1))) - -method log*(logger: RollingFileLogger, level: Level, args: varargs[string, `$`]) = - ## Logs to a file using rolling ``logger`` only. - if level >= logging.level and level >= logger.levelThreshold: - if logger.curLine >= logger.maxLines: - logger.file.close() - rotate(logger) - logger.logFiles.inc - logger.curLine = 0 - logger.file = open(logger.baseName, logger.baseMode, bufSize = logger.bufSize) - - writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) - if level in {lvlError, lvlFatal}: flushFile(logger.file) - logger.curLine.inc +when not defined(js): + method log*(logger: FileLogger, level: Level, args: varargs[string, `$`]) = + ## Logs to a file using ``logger`` only. + if level >= logging.level and level >= logger.levelThreshold: + writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) + if level in {lvlError, lvlFatal}: flushFile(logger.file) + + proc defaultFilename*(): string = + ## Returns the default filename for a logger. + var (path, name, _) = splitFile(getAppFilename()) + result = changeFileExt(path / name, "log") + + proc newFileLogger*(filename = defaultFilename(), + mode: FileMode = fmAppend, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + bufSize: int = -1): FileLogger = + ## Creates a new file logger. This logger logs to a file. + ## Use ``bufSize`` as size of the output buffer when writing the file + ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). + new(result) + result.levelThreshold = levelThreshold + result.file = open(filename, mode, bufSize = bufSize) + result.fmtStr = fmtStr + + # ------ + + proc countLogLines(logger: RollingFileLogger): int = + result = 0 + for line in logger.file.lines(): + result.inc() + + proc countFiles(filename: string): int = + # Example: file.log.1 + result = 0 + let (dir, name, ext) = splitFile(filename) + for kind, path in walkDir(dir): + if kind == pcFile: + let llfn = name & ext & ExtSep + if path.extractFilename.startsWith(llfn): + let numS = path.extractFilename[llfn.len .. ^1] + try: + let num = parseInt(numS) + if num > result: + result = num + except ValueError: discard + + proc newRollingFileLogger*(filename = defaultFilename(), + mode: FileMode = fmReadWrite, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr, + maxLines = 1000, + bufSize: int = -1): RollingFileLogger = + ## Creates a new rolling file logger. Once a file reaches ``maxLines`` lines + ## a new log file will be started and the old will be renamed. + ## Use ``bufSize`` as size of the output buffer when writing the file + ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). + new(result) + result.levelThreshold = levelThreshold + result.fmtStr = fmtStr + result.maxLines = maxLines + result.bufSize = bufSize + result.file = open(filename, mode, bufSize=result.bufSize) + result.curLine = 0 + result.baseName = filename + result.baseMode = mode + + result.logFiles = countFiles(filename) + + if mode == fmAppend: + # We need to get a line count because we will be appending to the file. + result.curLine = countLogLines(result) + + proc rotate(logger: RollingFileLogger) = + let (dir, name, ext) = splitFile(logger.baseName) + for i in countdown(logger.logFiles, 0): + let srcSuff = if i != 0: ExtSep & $i else: "" + moveFile(dir / (name & ext & srcSuff), + dir / (name & ext & ExtSep & $(i+1))) + + method log*(logger: RollingFileLogger, level: Level, args: varargs[string, `$`]) = + ## Logs to a file using rolling ``logger`` only. + if level >= logging.level and level >= logger.levelThreshold: + if logger.curLine >= logger.maxLines: + logger.file.close() + rotate(logger) + logger.logFiles.inc + logger.curLine = 0 + logger.file = open(logger.baseName, logger.baseMode, bufSize = logger.bufSize) + + writeLine(logger.file, substituteLog(logger.fmtStr, level, args)) + if level in {lvlError, lvlFatal}: flushFile(logger.file) + logger.curLine.inc # -------- @@ -323,10 +336,11 @@ proc getLogFilter*(): Level = when not defined(testing) and isMainModule: var L = newConsoleLogger() - var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) - var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) + when not defined(js): + var fL = newFileLogger("test.log", fmtStr = verboseFmtStr) + var rL = newRollingFileLogger("rolling.log", fmtStr = verboseFmtStr) + addHandler(fL) + addHandler(rL) addHandler(L) - addHandler(fL) - addHandler(rL) for i in 0 .. 25: info("hello", i) diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 134581a06..36e6cf52f 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -31,7 +31,7 @@ ## ## **Note**: The ``to`` and ``$$`` operations are available at compile-time! -import streams, typeinfo, json, intsets, tables +import streams, typeinfo, json, intsets, tables, unicode proc ptrToInt(x: pointer): int {.inline.} = result = cast[int](x) # don't skip alignment @@ -92,7 +92,15 @@ proc storeAny(s: Stream, a: Any, stored: var IntSet) = of akString: var x = getString(a) if isNil(x): s.write("null") - else: s.write(escapeJson(x)) + elif x.validateUtf8() == -1: s.write(escapeJson(x)) + else: + s.write("[") + var i = 0 + for c in x: + if i > 0: s.write(", ") + s.write($ord(c)) + inc(i) + s.write("]") of akInt..akInt64, akUInt..akUInt64: s.write($getBiggestInt(a)) of akFloat..akFloat128: s.write($getBiggestFloat(a)) @@ -207,6 +215,18 @@ proc loadAny(p: var JsonParser, a: Any, t: var Table[BiggestInt, pointer]) = of jsonString: setString(a, p.str) next(p) + of jsonArrayStart: + next(p) + var str = "" + while p.kind == jsonInt: + let code = p.getInt() + if code < 0 or code > 255: + raiseParseErr(p, "invalid charcode: " & $code) + str.add(chr(code)) + next(p) + if p.kind == jsonArrayEnd: next(p) + else: raiseParseErr(p, "an array of charcodes expected for string") + setString(a, str) else: raiseParseErr(p, "string expected") of akInt..akInt64, akUInt..akUInt64: if p.kind == jsonInt: diff --git a/lib/pure/net.nim b/lib/pure/net.nim index bd208761b..d4f239c49 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -106,9 +106,13 @@ when defineSsl: {.deprecated: [ESSL: SSLError, TSSLCVerifyMode: SSLCVerifyMode, TSSLProtVersion: SSLProtVersion, PSSLContext: SSLContext, TSSLAcceptResult: SSLAcceptResult].} +else: + type + SslContext* = void # TODO: Workaround #4797. const BufferSize*: int = 4000 ## size of a buffered socket's buffer + MaxLineLength* = 1_000_000 type SocketImpl* = object ## socket type @@ -206,7 +210,7 @@ proc newSocket*(fd: SocketHandle, domain: Domain = AF_INET, result.currPos = 0 # Set SO_NOSIGPIPE on OS X. - when defined(macosx): + when defined(macosx) and not defined(nimdoc): setSockOptInt(fd, SOL_SOCKET, SO_NOSIGPIPE, 1) proc newSocket*(domain, sockType, protocol: cint, buffered = true): Socket = @@ -966,6 +970,22 @@ proc recv*(socket: Socket, data: var string, size: int, timeout = -1, socket.socketError(result, lastError = lastError) data.setLen(result) +proc recv*(socket: Socket, size: int, timeout = -1, + flags = {SocketFlag.SafeDisconn}): string {.inline.} = + ## Higher-level version of ``recv`` which returns a string. + ## + ## When ``""`` is returned the socket's connection has been closed. + ## + ## This function will throw an EOS exception when an error occurs. + ## + ## A timeout may be specified in milliseconds, if enough data is not received + ## within the time specified an ETimeout exception will be raised. + ## + ## + ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + result = newString(size) + discard recv(socket, result, size, timeout, flags) + proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = if socket.isBuffered: result = 1 @@ -987,7 +1007,7 @@ proc peekChar(socket: Socket, c: var char): int {.tags: [ReadIOEffect].} = result = recv(socket.fd, addr(c), 1, MSG_PEEK) proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, - flags = {SocketFlag.SafeDisconn}) {. + flags = {SocketFlag.SafeDisconn}, maxLength = MaxLineLength) {. tags: [ReadIOEffect, TimeEffect].} = ## Reads a line of data from ``socket``. ## @@ -1002,6 +1022,10 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, ## A timeout can be specified in milliseconds, if data is not received within ## the specified time an ETimeout exception will be raised. ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. template addNLIfEmpty() = @@ -1035,6 +1059,36 @@ proc readLine*(socket: Socket, line: var TaintedString, timeout = -1, return add(line.string, c) + # Verify that this isn't a DOS attack: #3847. + if line.string.len > maxLength: + let msg = "recvLine received more than the specified `maxLength` " & + "allowed." + raise newException(ValueError, msg) + +proc recvLine*(socket: Socket, timeout = -1, + flags = {SocketFlag.SafeDisconn}, + maxLength = MaxLineLength): TaintedString = + ## Reads a line of data from ``socket``. + ## + ## If a full line is read ``\r\L`` is not + ## added to the result, however if solely ``\r\L`` is read then the result + ## will be set to it. + ## + ## If the socket is disconnected, the result will be set to ``""``. + ## + ## An EOS exception will be raised in the case of a socket error. + ## + ## A timeout can be specified in milliseconds, if data is not received within + ## the specified time an ETimeout exception will be raised. + ## + ## The ``maxLength`` parameter determines the maximum amount of characters + ## that can be read before a ``ValueError`` is raised. This prevents Denial + ## of Service (DOS) attacks. + ## + ## **Warning**: Only the ``SafeDisconn`` flag is currently supported. + result = "" + readLine(socket, result, timeout, flags, maxLength) + proc recvFrom*(socket: Socket, data: var string, length: int, address: var string, port: var Port, flags = 0'i32): int {. tags: [ReadIOEffect].} = diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 1e474f4d4..cdbe170cc 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -580,9 +580,9 @@ proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", when useWinUnicode: let s = newWideCString(source) let d = newWideCString(dest) - if moveFileW(s, d, 0'i32) == 0'i32: raiseOSError(osLastError()) + if moveFileW(s, d) == 0'i32: raiseOSError(osLastError()) else: - if moveFileA(source, dest, 0'i32) == 0'i32: raiseOSError(osLastError()) + if moveFileA(source, dest) == 0'i32: raiseOSError(osLastError()) else: if c_rename(source, dest) != 0'i32: raiseOSError(osLastError(), $strerror(errno)) @@ -595,12 +595,12 @@ when not declared(ENOENT) and not defined(Windows): when defined(Windows): when useWinUnicode: - template deleteFile(file: expr): expr {.immediate.} = deleteFileW(file) - template setFileAttributes(file, attrs: expr): expr {.immediate.} = + template deleteFile(file: untyped): untyped = deleteFileW(file) + template setFileAttributes(file, attrs: untyped): untyped = setFileAttributesW(file, attrs) else: - template deleteFile(file: expr): expr {.immediate.} = deleteFileA(file) - template setFileAttributes(file, attrs: expr): expr {.immediate.} = + template deleteFile(file: untyped): untyped = deleteFileA(file) + template setFileAttributes(file, attrs: untyped): untyped = setFileAttributesA(file, attrs) proc removeFile*(file: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = @@ -865,6 +865,20 @@ type {.deprecated: [TPathComponent: PathComponent].} + +when defined(posix): + proc getSymlinkFileKind(path: string): PathComponent = + # Helper function. + var s: Stat + assert(path != "") + if stat(path, s) < 0'i32: + raiseOSError(osLastError()) + if S_ISDIR(s.st_mode): + result = pcLinkToDir + else: + result = pcLinkToFile + + proc staticWalkDir(dir: string; relative: bool): seq[ tuple[kind: PathComponent, path: string]] = discard @@ -931,13 +945,15 @@ iterator walkDir*(dir: string; relative=false): tuple[kind: PathComponent, path: if x.d_type == DT_DIR: k = pcDir if x.d_type == DT_LNK: if dirExists(y): k = pcLinkToDir - else: k = succ(k) + else: k = pcLinkToFile yield (k, y) continue if lstat(y, s) < 0'i32: break - if S_ISDIR(s.st_mode): k = pcDir - if S_ISLNK(s.st_mode): k = succ(k) + if S_ISDIR(s.st_mode): + k = pcDir + elif S_ISLNK(s.st_mode): + k = getSymlinkFileKind(y) yield (k, y) iterator walkDirRec*(dir: string, filter={pcFile, pcDir}): string {. @@ -1047,7 +1063,7 @@ proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", proc createSymlink*(src, dest: string) = ## Create a symbolic link at `dest` which points to the item specified - ## by `src`. On most operating systems, will fail if a lonk + ## by `src`. On most operating systems, will fail if a link already exists. ## ## **Warning**: ## Some OS's (such as Microsoft Windows) restrict the creation @@ -1070,8 +1086,8 @@ proc createHardlink*(src, dest: string) = ## Create a hard link at `dest` which points to the item specified ## by `src`. ## - ## **Warning**: Most OS's restrict the creation of hard links to - ## root users (administrators) . + ## **Warning**: Some OS's restrict the creation of hard links to + ## root users (administrators). when defined(Windows): when useWinUnicode: var wSrc = newWideCString(src) @@ -1502,7 +1518,7 @@ type lastWriteTime*: Time # Time file was last modified/written to. creationTime*: Time # Time file was created. Not supported on all systems! -template rawToFormalFileInfo(rawInfo, formalInfo): untyped = +template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped = ## Transforms the native file info structure into the one nim uses. ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows, ## or a 'Stat' structure on posix @@ -1557,8 +1573,11 @@ template rawToFormalFileInfo(rawInfo, formalInfo): untyped = checkAndIncludeMode(S_IXOTH, fpOthersExec) formalInfo.kind = pcFile - if S_ISDIR(rawInfo.st_mode): formalInfo.kind = pcDir - if S_ISLNK(rawInfo.st_mode): formalInfo.kind.inc() + if S_ISDIR(rawInfo.st_mode): + formalInfo.kind = pcDir + elif S_ISLNK(rawInfo.st_mode): + assert(path != "") # symlinks can't occur for file handles + formalInfo.kind = getSymlinkFileKind(path) proc getFileInfo*(handle: FileHandle): FileInfo = ## Retrieves file information for the file object represented by the given @@ -1574,12 +1593,12 @@ proc getFileInfo*(handle: FileHandle): FileInfo = var realHandle = get_osfhandle(handle) if getFileInformationByHandle(realHandle, addr rawInfo) == 0: raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + rawToFormalFileInfo(rawInfo, "", result) else: var rawInfo: Stat if fstat(handle, rawInfo) < 0'i32: raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + rawToFormalFileInfo(rawInfo, "", result) proc getFileInfo*(file: File): FileInfo = if file.isNil: @@ -1608,7 +1627,7 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo = raiseOSError(osLastError()) if getFileInformationByHandle(handle, addr rawInfo) == 0: raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + rawToFormalFileInfo(rawInfo, path, result) discard closeHandle(handle) else: var rawInfo: Stat @@ -1618,7 +1637,7 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo = else: if lstat(path, rawInfo) < 0'i32: raiseOSError(osLastError()) - rawToFormalFileInfo(rawInfo, result) + rawToFormalFileInfo(rawInfo, path, result) proc isHidden*(path: string): bool = ## Determines whether a given path is hidden or not. Returns false if the diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 7378520e3..6c2debb1b 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -48,7 +48,7 @@ type inHandle, outHandle, errHandle: FileHandle inStream, outStream, errStream: Stream id: Pid - exitCode: cint + exitStatus: cint options: set[ProcessOption] Process* = ref ProcessObj ## represents an operating system process @@ -731,7 +731,7 @@ elif not defined(useNimRtl): pStdin, pStdout, pStderr: array[0..1, cint] new(result) result.options = options - result.exitCode = -3 # for ``waitForExit`` + result.exitStatus = -3 # for ``waitForExit`` if poParentStreams notin options: if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or pipe(pStderr) != 0'i32: @@ -957,13 +957,10 @@ elif not defined(useNimRtl): proc running(p: Process): bool = var ret : int - when not defined(freebsd): - ret = waitpid(p.id, p.exitCode, WNOHANG) - else: - var status : cint = 1 - ret = waitpid(p.id, status, WNOHANG) - if WIFEXITED(status): - p.exitCode = status + var status : cint = 1 + ret = waitpid(p.id, status, WNOHANG) + if WIFEXITED(status): + p.exitStatus = status if ret == 0: return true # Can't establish status. Assume running. result = ret == int(p.id) @@ -980,11 +977,12 @@ elif not defined(useNimRtl): import kqueue, times proc waitForExit(p: Process, timeout: int = -1): int = - if p.exitCode != -3: return p.exitCode + if p.exitStatus != -3: return int(p.exitStatus) shr 8 if timeout == -1: - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + var status : cint = 1 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status else: var kqFD = kqueue() if kqFD == -1: @@ -1004,6 +1002,7 @@ elif not defined(useNimRtl): try: while true: + var status : cint = 1 var count = kevent(kqFD, addr(kevIn), 1, addr(kevOut), 1, addr(tmspec)) if count < 0: @@ -1014,22 +1013,22 @@ elif not defined(useNimRtl): # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status break else: if kevOut.ident == p.id.uint and kevOut.filter == EVFILT_PROC: - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status break else: raiseOSError(osLastError()) finally: discard posix.close(kqFD) - result = int(p.exitCode) shr 8 + result = int(p.exitStatus) shr 8 else: import times @@ -1061,15 +1060,16 @@ elif not defined(useNimRtl): s.tv_sec = b.tv_sec s.tv_nsec = b.tv_nsec - #if waitPid(p.id, p.exitCode, 0) == int(p.id): + #if waitPid(p.id, p.exitStatus, 0) == int(p.id): # ``waitPid`` fails if the process is not running anymore. But then - # ``running`` probably set ``p.exitCode`` for us. Since ``p.exitCode`` is + # ``running`` probably set ``p.exitStatus`` for us. Since ``p.exitStatus`` is # initialized with -3, wrong success exit codes are prevented. - if p.exitCode != -3: return p.exitCode + if p.exitStatus != -3: return int(p.exitStatus) shr 8 if timeout == -1: - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + var status : cint = 1 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status else: var nmask, omask: Sigset var sinfo: SigInfo @@ -1100,9 +1100,10 @@ elif not defined(useNimRtl): let res = sigtimedwait(nmask, sinfo, tmspec) if res == SIGCHLD: if sinfo.si_pid == p.id: - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + var status : cint = 1 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status break else: # we have SIGCHLD, but not for process we are waiting, @@ -1122,9 +1123,10 @@ elif not defined(useNimRtl): # timeout expired, so we trying to kill process if posix.kill(p.id, SIGKILL) == -1: raiseOSError(osLastError()) - if waitpid(p.id, p.exitCode, 0) < 0: - p.exitCode = -3 + var status : cint = 1 + if waitpid(p.id, status, 0) < 0: raiseOSError(osLastError()) + p.exitStatus = status break else: raiseOSError(err) @@ -1136,15 +1138,19 @@ elif not defined(useNimRtl): if sigprocmask(SIG_UNBLOCK, nmask, omask) == -1: raiseOSError(osLastError()) - result = int(p.exitCode) shr 8 + result = int(p.exitStatus) shr 8 proc peekExitCode(p: Process): int = - if p.exitCode != -3: return p.exitCode - var ret = waitpid(p.id, p.exitCode, WNOHANG) + var status : cint = 1 + if p.exitStatus != -3: return int(p.exitStatus) shr 8 + var ret = waitpid(p.id, status, WNOHANG) var b = ret == int(p.id) if b: result = -1 - if not WIFEXITED(p.exitCode): result = -1 - else: result = p.exitCode.int shr 8 + if WIFEXITED(status): + p.exitStatus = status + result = p.exitStatus.int shr 8 + else: + result = -1 proc createStream(stream: var Stream, handle: var FileHandle, fileMode: FileMode) = diff --git a/lib/pure/rationals.nim b/lib/pure/rationals.nim index bf134f2ae..c2ba2b1f3 100644 --- a/lib/pure/rationals.nim +++ b/lib/pure/rationals.nim @@ -349,4 +349,4 @@ when isMainModule: assert toRational(0.98765432) == 12345679 // 12500000 assert toRational(0.1, 1000000) == 1 // 10 assert toRational(0.9, 1000000) == 9 // 10 - assert toRational(PI) == 80143857 // 25510582 + #assert toRational(PI) == 80143857 // 25510582 diff --git a/lib/pure/securehash.nim b/lib/pure/securehash.nim index a30fad92a..1f00ce8d3 100644 --- a/lib/pure/securehash.nim +++ b/lib/pure/securehash.nim @@ -15,30 +15,6 @@ type Sha1Digest = array[0 .. Sha1DigestSize-1, uint8] SecureHash* = distinct Sha1Digest -proc sha1(src: string) : Sha1Digest - -proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) -proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) -proc `$`*(self: SecureHash): string = - result = "" - for v in Sha1Digest(self): - result.add(toHex(int(v), 2)) - -proc parseSecureHash*(hash: string): SecureHash = - for i in 0.. <Sha1DigestSize: - Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) - -proc `==`*(a, b: SecureHash): bool = - # Not a constant-time comparison, but that's acceptable in this context - Sha1Digest(a) == Sha1Digest(b) - - -when isMainModule: - let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") - doAssert hash1 == hash1 - doAssert parseSecureHash($hash1) == hash1 - - # Copyright (c) 2011, Micael Hildenborg # All rights reserved. # @@ -90,10 +66,10 @@ proc innerHash(state: var Sha1State, w: var Sha1Buffer) = var round = 0 - template rot(value, bits: uint32): uint32 {.immediate.} = + template rot(value, bits: uint32): uint32 = (value shl bits) or (value shr (32 - bits)) - template sha1(fun, val: uint32): stmt = + template sha1(fun, val: uint32) = let t = rot(a, 5) + fun + e + val + w[round] e = d d = c @@ -101,12 +77,12 @@ proc innerHash(state: var Sha1State, w: var Sha1Buffer) = b = a a = t - template process(body: stmt): stmt = + template process(body: untyped) = w[round] = rot(w[round - 3] xor w[round - 8] xor w[round - 14] xor w[round - 16], 1) body inc(round) - template wrap(dest, value: expr): stmt {.immediate.} = + template wrap(dest, value: untyped) = let v = dest + value dest = v @@ -136,7 +112,7 @@ proc innerHash(state: var Sha1State, w: var Sha1Buffer) = wrap state[3], d wrap state[4], e -template computeInternal(src: expr): stmt {.immediate.} = +template computeInternal(src: untyped) = #Initialize state var state: Sha1State init(state) @@ -196,3 +172,24 @@ template computeInternal(src: expr): stmt {.immediate.} = proc sha1(src: string) : Sha1Digest = ## Calculate SHA1 from input string computeInternal(src) + +proc secureHash*(str: string): SecureHash = SecureHash(sha1(str)) +proc secureHashFile*(filename: string): SecureHash = secureHash(readFile(filename)) +proc `$`*(self: SecureHash): string = + result = "" + for v in Sha1Digest(self): + result.add(toHex(int(v), 2)) + +proc parseSecureHash*(hash: string): SecureHash = + for i in 0.. <Sha1DigestSize: + Sha1Digest(result)[i] = uint8(parseHexInt(hash[i*2] & hash[i*2 + 1])) + +proc `==`*(a, b: SecureHash): bool = + # Not a constant-time comparison, but that's acceptable in this context + Sha1Digest(a) == Sha1Digest(b) + + +when isMainModule: + let hash1 = secureHash("a93tgj0p34jagp9[agjp98ajrhp9aej]") + doAssert hash1 == hash1 + doAssert parseSecureHash($hash1) == hash1 diff --git a/lib/pure/selectors.nim b/lib/pure/selectors.nim index 098b78c95..506b2cec0 100644 --- a/lib/pure/selectors.nim +++ b/lib/pure/selectors.nim @@ -375,6 +375,11 @@ proc contains*(s: Selector, key: SelectorKey): bool = when not defined(nimdoc): return key.fd in s and s.fds[key.fd] == key +proc len*(s: Selector): int = + ## Retrieves the number of registered file descriptors in this Selector. + when not defined(nimdoc): + return s.fds.len + {.deprecated: [TEvent: Event, PSelectorKey: SelectorKey, TReadyInfo: ReadyInfo, PSelector: Selector].} diff --git a/lib/pure/stats.nim b/lib/pure/stats.nim index ec4cd182b..2004337df 100644 --- a/lib/pure/stats.nim +++ b/lib/pure/stats.nim @@ -334,15 +334,17 @@ when isMainModule: doAssert(rs1.sum == 9.5) doAssert(rs1.mean() == 2.375) - var rr: RunningRegress - rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0]) - doAssert(rr.slope() == 0.9695585996955861) - doAssert(rr.intercept() == -0.03424657534246611) - doAssert(rr.correlation() == 0.9905100362239381) - var rr1, rr2: RunningRegress - rr1.push(@[0.0,1.0], @[0.0,1.0]) - rr2.push(@[2.8,3.0,4.0], @[2.3,3.0,4.0]) - let rr3 = rr1 + rr2 - doAssert(rr3.correlation() == rr.correlation()) - doAssert(clean(rr3.slope()) == clean(rr.slope())) - doAssert(clean(rr3.intercept()) == clean(rr.intercept())) + when not defined(cpu32): + # XXX For some reason on 32bit CPUs these results differ + var rr: RunningRegress + rr.push(@[0.0,1.0,2.8,3.0,4.0], @[0.0,1.0,2.3,3.0,4.0]) + doAssert(rr.slope() == 0.9695585996955861) + doAssert(rr.intercept() == -0.03424657534246611) + doAssert(rr.correlation() == 0.9905100362239381) + var rr1, rr2: RunningRegress + rr1.push(@[0.0,1.0], @[0.0,1.0]) + rr2.push(@[2.8,3.0,4.0], @[2.3,3.0,4.0]) + let rr3 = rr1 + rr2 + doAssert(rr3.correlation() == rr.correlation()) + doAssert(clean(rr3.slope()) == clean(rr.slope())) + doAssert(clean(rr3.intercept()) == clean(rr.intercept())) diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 1c761cd92..c5d471dfa 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -13,9 +13,16 @@ ## for the string table is also provided. import - os, hashes, strutils + hashes, strutils -include "system/inclrtl" +when defined(js): + {.pragma: rtlFunc.} + {.pragma: deprecatedGetFunc.} +else: + {.pragma: deprecatedGetFunc, deprecatedGet.} + {.pragma: rtlFunc, rtl.} + import os + include "system/inclrtl" type StringTableMode* = enum ## describes the tables operation mode @@ -34,7 +41,7 @@ type {.deprecated: [TStringTableMode: StringTableMode, TStringTable: StringTableObj, PStringTable: StringTableRef].} -proc len*(t: StringTableRef): int {.rtl, extern: "nst$1".} = +proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = ## returns the number of keys in `t`. result = t.counter @@ -59,7 +66,7 @@ iterator values*(t: StringTableRef): string = type FormatFlag* = enum ## flags for the `%` operator useEnvironment, ## use environment variable if the ``$key`` - ## is not found in the table + ## is not found in the table. Does nothing when using `js` target. useEmpty, ## use the empty string as a default, thus it ## won't throw an exception if ``$key`` is not ## in the table @@ -111,7 +118,7 @@ template get(t: StringTableRef, key: string): stmt {.immediate.} = raise newException(KeyError, "key not found") proc `[]`*(t: StringTableRef, key: string): var string {. - rtl, extern: "nstTake", deprecatedGet.} = + rtlFunc, extern: "nstTake", deprecatedGetFunc.} = ## retrieves the location at ``t[key]``. If `key` is not in `t`, the ## ``KeyError`` exception is raised. One can check with ``hasKey`` whether ## the key exists. @@ -127,7 +134,7 @@ proc getOrDefault*(t: StringTableRef; key: string): string = if index >= 0: result = t.data[index].val else: result = "" -proc hasKey*(t: StringTableRef, key: string): bool {.rtl, extern: "nst$1".} = +proc hasKey*(t: StringTableRef, key: string): bool {.rtlFunc, extern: "nst$1".} = ## returns true iff `key` is in the table `t`. result = rawGet(t, key) >= 0 @@ -145,7 +152,7 @@ proc enlarge(t: StringTableRef) = if not isNil(t.data[i].key): rawInsert(t, n, t.data[i].key, t.data[i].val) swap(t.data, n) -proc `[]=`*(t: StringTableRef, key, val: string) {.rtl, extern: "nstPut".} = +proc `[]=`*(t: StringTableRef, key, val: string) {.rtlFunc, extern: "nstPut".} = ## puts a (key, value)-pair into `t`. var index = rawGet(t, key) if index >= 0: @@ -164,14 +171,17 @@ proc raiseFormatException(s: string) = proc getValue(t: StringTableRef, flags: set[FormatFlag], key: string): string = if hasKey(t, key): return t.getOrDefault(key) # hm difficult: assume safety in taint mode here. XXX This is dangerous! - if useEnvironment in flags: result = os.getEnv(key).string - else: result = "" + when defined(js): + result = "" + else: + if useEnvironment in flags: result = os.getEnv(key).string + else: result = "" if result.len == 0: if useKey in flags: result = '$' & key elif useEmpty notin flags: raiseFormatException(key) proc newStringTable*(mode: StringTableMode): StringTableRef {. - rtl, extern: "nst$1".} = + rtlFunc, extern: "nst$1".} = ## creates a new string table that is empty. new(result) result.mode = mode @@ -189,7 +199,7 @@ proc clear*(s: StringTableRef, mode: StringTableMode) = proc newStringTable*(keyValuePairs: varargs[string], mode: StringTableMode): StringTableRef {. - rtl, extern: "nst$1WithPairs".} = + rtlFunc, extern: "nst$1WithPairs".} = ## creates a new string table with given key value pairs. ## Example:: ## var mytab = newStringTable("key1", "val1", "key2", "val2", @@ -202,7 +212,7 @@ proc newStringTable*(keyValuePairs: varargs[string], proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], mode: StringTableMode = modeCaseSensitive): StringTableRef {. - rtl, extern: "nst$1WithTableConstr".} = + rtlFunc, extern: "nst$1WithTableConstr".} = ## creates a new string table with given key value pairs. ## Example:: ## var mytab = newStringTable({"key1": "val1", "key2": "val2"}, @@ -211,7 +221,7 @@ proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], for key, val in items(keyValuePairs): result[key] = val proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. - rtl, extern: "nstFormat".} = + rtlFunc, extern: "nstFormat".} = ## The `%` operator for string tables. const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} @@ -240,7 +250,7 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {. add(result, f[i]) inc(i) -proc `$`*(t: StringTableRef): string {.rtl, extern: "nstDollar".} = +proc `$`*(t: StringTableRef): string {.rtlFunc, extern: "nstDollar".} = ## The `$` operator for string tables. if t.len == 0: result = "{:}" diff --git a/lib/pure/terminal.nim b/lib/pure/terminal.nim index 1f34ec07e..d4734c3e3 100644 --- a/lib/pure/terminal.nim +++ b/lib/pure/terminal.nim @@ -60,6 +60,21 @@ when defined(windows): lpConsoleScreenBufferInfo: ptr CONSOLE_SCREEN_BUFFER_INFO): WINBOOL{.stdcall, dynlib: "kernel32", importc: "GetConsoleScreenBufferInfo".} + proc terminalWidthIoctl*(handles: openArray[Handle]): int = + var csbi: CONSOLE_SCREEN_BUFFER_INFO + for h in handles: + if getConsoleScreenBufferInfo(h, addr csbi) != 0: + return int(csbi.srWindow.Right - csbi.srWindow.Left + 1) + return 0 + + proc terminalWidth*(): int = + var w: int = 0 + w = terminalWidthIoctl([ getStdHandle(STD_INPUT_HANDLE), + getStdHandle(STD_OUTPUT_HANDLE), + getStdHandle(STD_ERROR_HANDLE) ] ) + if w > 0: return w + return 80 + proc setConsoleCursorPosition(hConsoleOutput: HANDLE, dwCursorPosition: COORD): WINBOOL{. stdcall, dynlib: "kernel32", importc: "SetConsoleCursorPosition".} @@ -123,7 +138,7 @@ when defined(windows): if f == stderr: hStderr else: hStdout else: - import termios + import termios, posix, os, parseutils proc setRaw(fd: FileHandle, time: cint = TCSAFLUSH) = var mode: Termios @@ -137,6 +152,33 @@ else: mode.c_cc[VTIME] = 0.cuchar discard fd.tcsetattr(time, addr mode) + proc terminalWidthIoctl*(fds: openArray[int]): int = + ## Returns terminal width from first fd that supports the ioctl. + + var win: IOctl_WinSize + for fd in fds: + if ioctl(cint(fd), TIOCGWINSZ, addr win) != -1: + return int(win.ws_col) + return 0 + + var L_ctermid{.importc, header: "<stdio.h>".}: cint + proc terminalWidth*(): int = + ## Returns some reasonable terminal width from either standard file + ## descriptors, controlling terminal, environment variables or tradition. + + var w = terminalWidthIoctl([0, 1, 2]) #Try standard file descriptors + if w > 0: return w + var cterm = newString(L_ctermid) #Try controlling tty + var fd = open(ctermid(cstring(cterm)), O_RDONLY) + if fd != -1: + w = terminalWidthIoctl([ int(fd) ]) + discard close(fd) + if w > 0: return w + var s = getEnv("COLUMNS") #Try standard env var + if len(s) > 0 and parseInt(string(s), w) > 0 and w > 0: + return w + return 80 #Finally default to venerable value + proc setCursorPos*(f: File, x, y: int) = ## Sets the terminal's cursor to the (x,y) position. ## (0,0) is the upper left of the screen. diff --git a/lib/pure/times.nim b/lib/pure/times.nim index d6eb29e1c..b78a2b966 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -1248,8 +1248,11 @@ proc parse*(value, layout: string): TimeInfo = else: parseToken(info, token, value, j) token = "" - # Reset weekday as it might not have been provided and the default may be wrong - info.weekday = getLocalTime(toTime(info)).weekday + # Reset weekday (might not have been provided and the default may be wrong) + # and yearday (is never provided directly and therefore probably wrong) + let processed = getLocalTime(toTime(info)) + info.weekday = processed.weekday + info.yearday = processed.yearday return info # Leap year calculations are adapted from: diff --git a/lib/pure/unicode.nim b/lib/pure/unicode.nim index 0cbe8de7a..6ba966816 100644 --- a/lib/pure/unicode.nim +++ b/lib/pure/unicode.nim @@ -24,7 +24,7 @@ proc `<=%`*(a, b: Rune): bool = return int(a) <=% int(b) proc `<%`*(a, b: Rune): bool = return int(a) <% int(b) proc `==`*(a, b: Rune): bool = return int(a) == int(b) -template ones(n: expr): expr = ((1 shl n)-1) +template ones(n: untyped): untyped = ((1 shl n)-1) proc runeLen*(s: string): int {.rtl, extern: "nuc$1".} = ## Returns the number of Unicode characters of the string ``s`` @@ -49,7 +49,7 @@ proc runeLenAt*(s: string, i: Natural): int = elif ord(s[i]) shr 1 == 0b1111110: result = 6 else: result = 1 -template fastRuneAt*(s: string, i: int, result: expr, doInc = true) = +template fastRuneAt*(s: string, i: int, result: untyped, doInc = true) = ## Returns the Unicode character ``s[i]`` in ``result``. If ``doInc == true`` ## ``i`` is incremented by the number of bytes that have been processed. bind ones @@ -1628,7 +1628,7 @@ proc reversed*(s: string): string = blockPos = 0 r: Rune - template reverseUntil(pos): stmt = + template reverseUntil(pos) = var j = pos - 1 while j > blockPos: result[newPos] = s[j] diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 92ddc3e75..0fc2e441e 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -77,6 +77,15 @@ checkpoints = @[] proc shouldRun(testName: string): bool = result = true +proc startSuite(name: string) = + template rawPrint() = echo("\n[Suite] ", name) + when not defined(ECMAScript): + if colorOutput: + styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name + else: rawPrint() + else: rawPrint() + + template suite*(name, body) {.dirty.} = ## Declare a test suite identified by `name` with optional ``setup`` ## and/or ``teardown`` section. @@ -103,9 +112,11 @@ template suite*(name, body) {.dirty.} = ## ## .. code-block:: ## - ## [OK] 2 + 2 = 4 - ## [OK] (2 + -2) != 4 + ## [Suite] test suite for addition + ## [OK] 2 + 2 = 4 + ## [OK] (2 + -2) != 4 block: + bind startSuite template setup(setupBody: untyped) {.dirty.} = var testSetupIMPLFlag = true template testSetupIMPL: untyped {.dirty.} = setupBody @@ -114,14 +125,16 @@ template suite*(name, body) {.dirty.} = var testTeardownIMPLFlag = true template testTeardownIMPL: untyped {.dirty.} = teardownBody + let testInSuiteImplFlag = true + startSuite name body -proc testDone(name: string, s: TestStatus) = +proc testDone(name: string, s: TestStatus, indent: bool) = if s == FAILED: programResult += 1 - + let prefix = if indent: " " else: "" if outputLevel != PRINT_NONE and (outputLevel == PRINT_ALL or s == FAILED): - template rawPrint() = echo("[", $s, "] ", name) + template rawPrint() = echo(prefix, "[", $s, "] ", name) when not defined(ECMAScript): if colorOutput and not defined(ECMAScript): var color = case s @@ -129,7 +142,7 @@ proc testDone(name: string, s: TestStatus) = of FAILED: fgRed of SKIPPED: fgYellow else: fgWhite - styledEcho styleBright, color, "[", $s, "] ", fgWhite, name + styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name else: rawPrint() else: @@ -168,7 +181,7 @@ template test*(name, body) {.dirty.} = fail() finally: - testDone name, testStatusIMPL + testDone name, testStatusIMPL, declared(testInSuiteImplFlag) proc checkpoint*(msg: string) = ## Set a checkpoint identified by `msg`. Upon test failure all @@ -198,8 +211,9 @@ template fail* = ## ## outputs "Checkpoint A" before quitting. bind checkpoints + let prefix = if declared(testInSuiteImplFlag): " " else: "" for msg in items(checkpoints): - echo msg + echo prefix, msg when not defined(ECMAScript): if abortOnError: quit(1) diff --git a/lib/pure/xmltree.nim b/lib/pure/xmltree.nim index a9fc8998a..3c6eb14e3 100644 --- a/lib/pure/xmltree.nim +++ b/lib/pure/xmltree.nim @@ -226,6 +226,18 @@ proc noWhitespace(n: XmlNode): bool = proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = ## adds the textual representation of `n` to `result`. + + proc addEscapedAttr(result: var string, s: string) = + # `addEscaped` alternative with less escaped characters. + # Only to be used for escaping attribute values enclosed in double quotes! + for c in items(s): + case c + of '<': result.add("<") + of '>': result.add(">") + of '&': result.add("&") + of '"': result.add(""") + else: result.add(c) + if n == nil: return case n.k of xnElement: @@ -236,7 +248,7 @@ proc add*(result: var string, n: XmlNode, indent = 0, indWidth = 2) = result.add(' ') result.add(key) result.add("=\"") - result.addEscaped(val) + result.addEscapedAttr(val) result.add('"') if n.len > 0: result.add('>') @@ -381,6 +393,5 @@ proc findAll*(n: XmlNode, tag: string): seq[XmlNode] = findAll(n, tag, result) when isMainModule: - let link = "http://nim-lang.org" - assert """<a href="""" & escape(link) & """">Nim rules.</a>""" == + assert """<a href="http://nim-lang.org">Nim rules.</a>""" == $(<>a(href="http://nim-lang.org", newText("Nim rules."))) diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index 4b0066ee8..bd1b78efe 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,6 @@ [Package] name = "stdlib" -version = "0.14.3" +version = "0.15.1" author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index 26feb5ce8..fae111ce2 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -226,6 +226,7 @@ proc high*[T](x: T): T {.magic: "High", noSideEffect.} ## var arr = [1,2,3,4,5,6,7] ## high(arr) #=> 6 ## high(2) #=> 9223372036854775807 + ## high(int) #=> 9223372036854775807 proc low*[T](x: T): T {.magic: "Low", noSideEffect.} ## returns the lowest possible index of an array, a sequence, a string or @@ -236,6 +237,7 @@ proc low*[T](x: T): T {.magic: "Low", noSideEffect.} ## var arr = [1,2,3,4,5,6,7] ## low(arr) #=> 0 ## low(2) #=> -9223372036854775808 + ## low(int) #=> -9223372036854775808 type range*{.magic: "Range".}[T] ## Generic type to construct range types. @@ -603,7 +605,7 @@ proc `<`*[T](x: Ordinal[T]): T {.magic: "UnaryLt", noSideEffect.} ## unary ``<`` that can be used for nice looking excluding ranges: ## ## .. code-block:: nim - ## for i in 0 .. <10: echo i + ## for i in 0 .. <10: echo i #=> 0 1 2 3 4 5 6 7 8 9 ## ## Semantically this is the same as ``pred``. @@ -889,8 +891,8 @@ proc `shl` *(x, y: int64): int64 {.magic: "ShlI", noSideEffect.} ## computes the `shift left` operation of `x` and `y`. ## ## .. code-block:: Nim - ## 1'i32 shl 4 == 0x0000_0010 - ## 1'i64 shl 4 == 0x0000_0000_0000_0010 + ## 1'i32 shl 4 == 0x0000_0010 + ## 1'i64 shl 4 == 0x0000_0000_0000_0010 proc `and` *(x, y: int): int {.magic: "BitandI", noSideEffect.} proc `and` *(x, y: int8): int8 {.magic: "BitandI", noSideEffect.} @@ -1313,20 +1315,20 @@ when defined(boehmgc): when taintMode: type TaintedString* = distinct string ## a distinct string type that - ## is `tainted`:idx:. It is an alias for + ## is `tainted`:idx:, see `taint mode + ## <manual.html#taint-mode>`_ for + ## details. It is an alias for ## ``string`` if the taint mode is not - ## turned on. Use the ``-d:taintMode`` - ## command line switch to turn the taint - ## mode on. + ## turned on. proc len*(s: TaintedString): int {.borrow.} else: type TaintedString* = string ## a distinct string type that - ## is `tainted`:idx:. It is an alias for + ## is `tainted`:idx:, see `taint mode + ## <manual.html#taint-mode>`_ for + ## details. It is an alias for ## ``string`` if the taint mode is not - ## turned on. Use the ``-d:taintMode`` - ## command line switch to turn the taint - ## mode on. + ## turned on. when defined(profiler): proc nimProfile() {.compilerProc, noinline.} @@ -1410,7 +1412,7 @@ proc del*[T](x: var seq[T], i: Natural) {.noSideEffect.} = ## This is an O(1) operation. ## ## .. code-block:: nim - ## var i = @[1,2,3,4,5] + ## var i = @[1, 2, 3, 4, 5] ## i.del(2) #=> @[1, 2, 5, 4] let xl = x.len - 1 shallowCopy(x[i], x[xl]) @@ -1421,7 +1423,7 @@ proc delete*[T](x: var seq[T], i: Natural) {.noSideEffect.} = ## This is an O(n) operation. ## ## .. code-block:: nim - ## var i = @[1,2,3,4,5] + ## var i = @[1, 2, 3, 4, 5] ## i.delete(2) #=> @[1, 2, 4, 5] template defaultImpl = let xl = x.len @@ -1440,8 +1442,8 @@ proc insert*[T](x: var seq[T], item: T, i = 0.Natural) {.noSideEffect.} = ## inserts `item` into `x` at position `i`. ## ## .. code-block:: nim - ## var i = @[1,2,3,4,5] - ## i.insert(2,4) #=> @[1, 2, 3, 4, 2, 5] + ## var i = @[1, 2, 3, 4, 5] + ## i.insert(2, 4) #=> @[1, 2, 3, 4, 2, 5] template defaultImpl = let xl = x.len setLen(x, xl+1) @@ -1465,8 +1467,8 @@ proc repr*[T](x: T): string {.magic: "Repr", noSideEffect.} ## debugging tool. ## ## .. code-block:: nim - ## var s: seq[string] = @["test2","test2"] - ## var i = @[1,2,3,4,5] + ## var s: seq[string] = @["test2", "test2"] + ## var i = @[1, 2, 3, 4, 5] ## repr(s) #=> 0x1055eb050[0x1055ec050"test2", 0x1055ec078"test2"] ## repr(i) #=> 0x1055ed050[1, 2, 3, 4, 5] @@ -1519,7 +1521,7 @@ type # these work for most platforms: ## This is the same as the type ``double`` in *C*. clongdouble* {.importc: "long double", nodecl.} = BiggestFloat ## This is the same as the type ``long double`` in *C*. - ## This C type is not supported by Nim's code generator + ## This C type is not supported by Nim's code generator. cuchar* {.importc: "unsigned char", nodecl.} = char ## This is the same as the type ``unsigned char`` in *C*. @@ -1822,18 +1824,15 @@ const NimMajor*: int = 0 ## is the major number of Nim's version. - NimMinor*: int = 14 + NimMinor*: int = 15 ## is the minor number of Nim's version. - NimPatch*: int = 3 + NimPatch*: int = 1 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch ## is the version of Nim as a string. -{.deprecated: [TEndian: Endianness, NimrodVersion: NimVersion, - NimrodMajor: NimMajor, NimrodMinor: NimMinor, NimrodPatch: NimPatch].} - # GC interface: when not defined(nimscript) and hasAlloc: @@ -2654,12 +2653,11 @@ when not defined(JS): #and not defined(nimscript): when defined(nimscript): proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} - ## Opens a file named `filename` for reading. - ## - ## Then calls `readAll <#readAll>`_ and closes the file afterwards. - ## Returns the string. Raises an IO exception in case of an error. If - ## you need to call this inside a compile time macro you can use - ## `staticRead <#staticRead>`_. + ## Opens a file named `filename` for reading, calls `readAll + ## <#readAll>`_ and closes the file afterwards. Returns the string. + ## Raises an IO exception in case of an error. If # you need to call + ## this inside a compile time macro you can use `staticRead + ## <#staticRead>`_. proc writeFile*(filename, content: string) {.tags: [WriteIOEffect], benign.} ## Opens a file named `filename` for writing. Then writes the @@ -2733,7 +2731,7 @@ when not defined(JS): #and not defined(nimscript): proc setStdIoUnbuffered*() {.tags: [], benign.} ## Configures `stdin`, `stdout` and `stderr` to be unbuffered. - proc close*(f: File) {.tags: [].} + proc close*(f: File) {.tags: [], gcsafe.} ## Closes the file. proc endOfFile*(f: File): bool {.tags: [], benign.} @@ -2795,7 +2793,7 @@ when not defined(JS): #and not defined(nimscript): proc writeLine*[Ty](f: File, x: varargs[Ty, `$`]) {.inline, tags: [WriteIOEffect], benign.} - ## writes the values `x` to `f` and then writes "\n". + ## writes the values `x` to `f` and then writes "\\n". ## May throw an IO exception. proc getFileSize*(f: File): int64 {.tags: [ReadIOEffect], benign.} @@ -2926,14 +2924,14 @@ when not defined(JS): #and not defined(nimscript): ## allows you to override the behaviour of your application when CTRL+C ## is pressed. Only one such hook is supported. - proc writeStackTrace*() {.tags: [WriteIOEffect].} + proc writeStackTrace*() {.tags: [WriteIOEffect], gcsafe.} ## writes the current stack trace to ``stderr``. This is only works ## for debug builds. when hostOS != "standalone": - proc getStackTrace*(): string + proc getStackTrace*(): string {.gcsafe.} ## gets the current stack trace. This only works for debug builds. - proc getStackTrace*(e: ref Exception): string + proc getStackTrace*(e: ref Exception): string {.gcsafe.} ## gets the stack trace associated with `e`, which is the stack that ## lead to the ``raise`` statement. This only works for debug builds. diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index bed9fd906..745bbbf62 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -101,8 +101,8 @@ type # shared: var - bottomData: AvlNode - bottom: PAvlNode + bottomData {.threadvar.}: AvlNode + bottom {.threadvar.}: PAvlNode {.push stack_trace: off.} proc initAllocator() = diff --git a/lib/system/atomics.nim b/lib/system/atomics.nim index a79dcb152..3a43729dc 100644 --- a/lib/system/atomics.nim +++ b/lib/system/atomics.nim @@ -14,6 +14,7 @@ const someGcc = defined(gcc) or defined(llvm_gcc) or defined(clang) when someGcc and hasThreadSupport: type AtomMemModel* = distinct cint + var ATOMIC_RELAXED* {.importc: "__ATOMIC_RELAXED", nodecl.}: AtomMemModel ## No barriers or synchronization. var ATOMIC_CONSUME* {.importc: "__ATOMIC_CONSUME", nodecl.}: AtomMemModel @@ -165,7 +166,8 @@ when someGcc and hasThreadSupport: template fence*() = atomicThreadFence(ATOMIC_SEQ_CST) elif defined(vcc) and hasThreadSupport: proc addAndFetch*(p: ptr int, val: int): int {. - importc: "NimXadd", nodecl.} + importc: "_InterlockedExchangeAdd", header: "<intrin.h>".} + proc fence*() {.importc: "_ReadWriteBarrier", header: "<intrin.h>".} else: @@ -176,6 +178,8 @@ else: proc atomicInc*(memLoc: var int, x: int = 1): int = when someGcc and hasThreadSupport: result = atomic_add_fetch(memLoc.addr, x, ATOMIC_RELAXED) + elif defined(vcc) and hasThreadSupport: + result = addAndFetch(memLoc.addr, x) else: inc(memLoc, x) result = memLoc @@ -186,17 +190,33 @@ proc atomicDec*(memLoc: var int, x: int = 1): int = result = atomic_sub_fetch(memLoc.addr, x, ATOMIC_RELAXED) else: result = atomic_add_fetch(memLoc.addr, -x, ATOMIC_RELAXED) + elif defined(vcc) and hasThreadSupport: + result = addAndFetch(memLoc.addr, -x) else: dec(memLoc, x) result = memLoc -when defined(windows) and not someGcc: - proc interlockedCompareExchange(p: pointer; exchange, comparand: int): int - {.importc: "InterlockedCompareExchange", header: "<windows.h>", cdecl.} +when defined(vcc): + proc interlockedCompareExchange64(p: pointer; exchange, comparand: int64): int64 + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} + proc interlockedCompareExchange32(p: pointer; exchange, comparand: int32): int32 + {.importc: "_InterlockedCompareExchange", header: "<intrin.h>".} + proc interlockedCompareExchange8(p: pointer; exchange, comparand: byte): byte + {.importc: "_InterlockedCompareExchange64", header: "<intrin.h>".} proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = - interlockedCompareExchange(p, cast[int](newValue), cast[int](oldValue)) != 0 - # XXX fix for 64 bit build + when sizeof(T) == 8: + interlockedCompareExchange64(p, cast[int64](newValue), cast[int64](oldValue)) == + cast[int64](oldValue) + elif sizeof(T) == 4: + interlockedCompareExchange32(p, cast[int32](newValue), cast[int32](oldValue)) == + cast[int32](oldValue) + elif sizeof(T) == 1: + interlockedCompareExchange8(p, cast[byte](newValue), cast[byte](oldValue)) == + cast[byte](oldValue) + else: + {.error: "invalid CAS instruction".} + elif defined(tcc) and not defined(windows): when defined(amd64): {.emit:""" @@ -237,7 +257,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal) return 1; } """.} - + proc tcc_cas(p: ptr int; oldValue, newValue: int): bool {.importc: "__tcc_cas", nodecl.} proc cas*[T: bool|int|ptr](p: ptr T; oldValue, newValue: T): bool = @@ -249,14 +269,14 @@ else: # XXX is this valid for 'int'? -when (defined(x86) or defined(amd64)) and someGcc: +when (defined(x86) or defined(amd64)) and defined(vcc): + proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} +elif (defined(x86) or defined(amd64)) and someGcc: proc cpuRelax* {.inline.} = {.emit: """asm volatile("pause" ::: "memory");""".} elif someGcc or defined(tcc): proc cpuRelax* {.inline.} = {.emit: """asm volatile("" ::: "memory");""".} -elif (defined(x86) or defined(amd64)) and defined(vcc): - proc cpuRelax* {.importc: "YieldProcessor", header: "<windows.h>".} elif defined(icl): proc cpuRelax* {.importc: "_mm_pause", header: "xmmintrin.h".} elif false: diff --git a/lib/system/avltree.nim b/lib/system/avltree.nim index d5c901542..50faada26 100644 --- a/lib/system/avltree.nim +++ b/lib/system/avltree.nim @@ -9,7 +9,7 @@ # not really an AVL tree anymore, but still balanced ... -template isBottom(n: PAvlNode): bool = n == bottom +template isBottom(n: PAvlNode): bool = n.link[0] == n proc lowGauge(n: PAvlNode): int = var it = n @@ -52,7 +52,7 @@ proc split(t: var PAvlNode) = inc t.level proc add(a: var MemRegion, t: var PAvlNode, key, upperBound: int) {.benign.} = - if t == bottom: + if t.isBottom: t = allocAvlNode(a, key, upperBound) else: if key <% t.key: @@ -65,14 +65,14 @@ proc add(a: var MemRegion, t: var PAvlNode, key, upperBound: int) {.benign.} = split(t) proc del(a: var MemRegion, t: var PAvlNode, x: int) {.benign.} = - if t == bottom: return + if isBottom(t): return a.last = t if x <% t.key: del(a, t.link[0], x) else: a.deleted = t del(a, t.link[1], x) - if t == a.last and a.deleted != bottom and x == a.deleted.key: + if t == a.last and not isBottom(a.deleted) and x == a.deleted.key: a.deleted.key = t.key a.deleted.upperBound = t.upperBound a.deleted = bottom diff --git a/lib/system/channels.nim b/lib/system/channels.nim index 68c0e32d2..4b8b895a5 100644 --- a/lib/system/channels.nim +++ b/lib/system/channels.nim @@ -52,6 +52,7 @@ proc deinitRawChannel(p: pointer) = proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, mode: LoadStoreMode) {.benign.} + proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, mode: LoadStoreMode) {.benign.} = var @@ -71,6 +72,9 @@ proc storeAux(dest, src: pointer, n: ptr TNimNode, t: PRawChannel, proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, mode: LoadStoreMode) = + template `+!`(p: pointer; x: int): pointer = + cast[pointer](cast[int](p) +% x) + var d = cast[ByteAddress](dest) s = cast[ByteAddress](src) @@ -93,7 +97,9 @@ proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, if s2 == nil: unsureAsgnRef(x, s2) else: - unsureAsgnRef(x, copyString(cast[NimString](s2))) + let y = copyDeepString(cast[NimString](s2)) + #echo "loaded ", cast[int](y), " ", cast[string](y) + unsureAsgnRef(x, y) dealloc(t.region, s2) of tySequence: var s2 = cast[PPointer](src)[] @@ -107,27 +113,27 @@ proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, else: sysAssert(dest != nil, "dest == nil") if mode == mStore: - x[] = alloc(t.region, seq.len *% mt.base.size +% GenericSeqSize) + x[] = alloc0(t.region, seq.len *% mt.base.size +% GenericSeqSize) else: unsureAsgnRef(x, newObj(mt, seq.len * mt.base.size + GenericSeqSize)) var dst = cast[ByteAddress](cast[PPointer](dest)[]) + var dstseq = cast[PGenericSeq](dst) + dstseq.len = seq.len + dstseq.reserved = seq.len for i in 0..seq.len-1: storeAux( cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), cast[pointer](cast[ByteAddress](s2) +% i *% mt.base.size +% GenericSeqSize), mt.base, t, mode) - var dstseq = cast[PGenericSeq](dst) - dstseq.len = seq.len - dstseq.reserved = seq.len if mode != mStore: dealloc(t.region, s2) of tyObject: - # copy type field: - var pint = cast[ptr PNimType](dest) - # XXX use dynamic type here! - pint[] = mt if mt.base != nil: storeAux(dest, src, mt.base, t, mode) + else: + # copy type field: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] storeAux(dest, src, mt.node, t, mode) of tyTuple: storeAux(dest, src, mt.node, t, mode) @@ -144,16 +150,24 @@ proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel, else: unsureAsgnRef(x, nil) else: + #let size = if mt.base.kind == tyObject: cast[ptr PNimType](s)[].size + # else: mt.base.size if mode == mStore: - x[] = alloc(t.region, mt.base.size) + let dyntype = when declared(usrToCell): usrToCell(s).typ + else: mt + let size = dyntype.base.size + # we store the real dynamic 'ref type' at offset 0, so that + # no information is lost + let a = alloc0(t.region, size+sizeof(pointer)) + x[] = a + cast[PPointer](a)[] = dyntype + storeAux(a +! sizeof(pointer), s, dyntype.base, t, mode) else: - # XXX we should use the dynamic type here too, but that is not stored - # in the inbox at all --> use source[]'s object type? but how? we need - # a tyRef to the object! - var obj = newObj(mt, mt.base.size) + let dyntype = cast[ptr PNimType](s)[] + var obj = newObj(dyntype, dyntype.base.size) unsureAsgnRef(x, obj) - storeAux(x[], s, mt.base, t, mode) - if mode != mStore: dealloc(t.region, s) + storeAux(x[], s +! sizeof(pointer), dyntype.base, t, mode) + dealloc(t.region, s) else: copyMem(dest, src, mt.size) # copy raw bits @@ -196,10 +210,8 @@ template sendImpl(q: expr) {.immediate.} = if q.mask == ChannelDeadMask: sysFatal(DeadThreadError, "cannot send message; thread died") acquireSys(q.lock) - var m: TMsg - shallowCopy(m, msg) var typ = cast[PNimType](getTypeInfo(msg)) - rawSend(q, addr(m), typ) + rawSend(q, unsafeAddr(msg), typ) q.elemType = typ releaseSys(q.lock) signalSysCond(q.cond) @@ -230,8 +242,10 @@ proc recv*[TMsg](c: var Channel[TMsg]): TMsg = proc tryRecv*[TMsg](c: var Channel[TMsg]): tuple[dataAvailable: bool, msg: TMsg] = - ## try to receives a message from the channel `c` if available. Otherwise - ## it returns ``(false, default(msg))``. + ## Tries to receive a message from the channel `c`, but this can fail + ## for all sort of reasons, including contention. If it fails, + ## it returns ``(false, default(msg))`` otherwise it + ## returns ``(true, msg)``. var q = cast[PRawChannel](addr(c)) if q.mask != ChannelDeadMask: if tryAcquireSys(q.lock): diff --git a/lib/system/deepcopy.nim b/lib/system/deepcopy.nim index 5445a067c..38cc8cbf3 100644 --- a/lib/system/deepcopy.nim +++ b/lib/system/deepcopy.nim @@ -32,12 +32,6 @@ proc genericDeepCopyAux(dest, src: pointer, n: ptr TNimNode) {.benign.} = genericDeepCopyAux(dest, src, m) of nkNone: sysAssert(false, "genericDeepCopyAux") -proc copyDeepString(src: NimString): NimString {.inline.} = - if src != nil: - result = rawNewStringNoInit(src.len) - result.len = src.len - copyMem(addr(result.data), addr(src.data), src.len + 1) - proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = var d = cast[ByteAddress](dest) @@ -70,10 +64,11 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = of tyObject: # we need to copy m_type field for tyObject, as it could be empty for # sequence reallocations: - var pint = cast[ptr PNimType](dest) - pint[] = cast[ptr PNimType](src)[] if mt.base != nil: genericDeepCopyAux(dest, src, mt.base) + else: + var pint = cast[ptr PNimType](dest) + pint[] = cast[ptr PNimType](src)[] genericDeepCopyAux(dest, src, mt.node) of tyTuple: genericDeepCopyAux(dest, src, mt.node) @@ -103,16 +98,16 @@ proc genericDeepCopyAux(dest, src: pointer, mt: PNimType) = else: let realType = x.typ let z = newObj(realType, realType.base.size) - unsureAsgnRef(cast[PPointer](dest), z) x.typ = cast[PNimType](cast[int](z) or 1) genericDeepCopyAux(z, s2, realType.base) x.typ = realType else: - let realType = mt - let z = newObj(realType, realType.base.size) + let size = if mt.base.kind == tyObject: cast[ptr PNimType](s2)[].size + else: mt.base.size + let z = newObj(mt, size) unsureAsgnRef(cast[PPointer](dest), z) - genericDeepCopyAux(z, s2, realType.base) + genericDeepCopyAux(z, s2, mt.base) of tyPtr: # no cycle check here, but also not really required let s2 = cast[PPointer](src)[] diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 0a994efac..fa997e982 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -26,6 +26,8 @@ proc nimLoadLibraryError(path: string) = stderr.rawWrite("could not load: ") stderr.rawWrite(path) stderr.rawWrite("\n") + when not(defined(nimDebugDlOpen)): + stderr.rawWrite("compile with -d:nimDebugDlOpen for more information\n") quit(1) proc procAddrError(name: cstring) {.noinline.} = @@ -74,7 +76,8 @@ when defined(posix): when defined(nimDebugDlOpen): let error = dlerror() if error != nil: - c_fprintf(c_stderr, "%s\n", error) + stderr.write(error) + stderr.rawWrite("\n") proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr = result = dlsym(lib, name) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index b89729850..dcf41b67d 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -11,7 +11,8 @@ # use the heap (and nor exceptions) do not include the GC or memory allocator. var - errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign.}) + errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign, + nimcall.}) ## Function that will be called ## instead of stdmsg.write when printing stacktrace. ## Unstable API. @@ -26,7 +27,7 @@ else: proc writeToStdErr(msg: cstring) = discard MessageBoxA(0, msg, nil, 0) -proc showErrorMessage(data: cstring) = +proc showErrorMessage(data: cstring) {.gcsafe.} = if errorMessageWriter != nil: errorMessageWriter($data) else: diff --git a/lib/system/gc.nim b/lib/system/gc.nim index ba6b2fcf9..11897ce80 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -116,6 +116,8 @@ template gcAssert(cond: bool, msg: string) = echo "[GCASSERT] ", msg GC_disable() writeStackTrace() + #var x: ptr int + #echo x[] quit 1 proc addZCT(s: var CellSeq, c: PCell) {.noinline.} = diff --git a/lib/system/gc2.nim b/lib/system/gc2.nim index 089c9c915..ce2bfc2ae 100644 --- a/lib/system/gc2.nim +++ b/lib/system/gc2.nim @@ -33,6 +33,9 @@ when withRealTime and not declared(getTicks): when defined(memProfiler): proc nimProfile(requestedSize: int) {.benign.} +when hasThreadSupport: + include sharedlist + type ObjectSpaceIter = object state: range[-1..0] @@ -96,7 +99,7 @@ type stat: GcStat additionalRoots: CellSeq # dummy roots for GC_ref/unref spaceIter: ObjectSpaceIter - dumpHeapFile: File # File that is used for GC_dumpHeap + pDumpHeapFile: pointer # File that is used for GC_dumpHeap when hasThreadSupport: toDispose: SharedList[pointer] @@ -612,6 +615,9 @@ template checkTime {.dirty.} = # ---------------- dump heap ---------------- +template dumpHeapFile(gch: var GcHeap): File = + cast[File](gch.pDumpHeapFile) + proc debugGraph(s: PCell) = c_fprintf(gch.dumpHeapFile, "child %p\n", s) @@ -625,7 +631,7 @@ proc GC_dumpHeap*(file: File) = ## Dumps the GCed heap's content to a file. Can be useful for ## debugging. Produces an undocumented text file format that ## can be translated into "dot" syntax via the "heapdump2dot" tool. - gch.dumpHeapFile = file + gch.pDumpHeapFile = file var spaceIter: ObjectSpaceIter var d = gch.decStack.d for i in 0 .. < gch.decStack.len: @@ -643,7 +649,7 @@ proc GC_dumpHeap*(file: File) = writeCell(file, "cell ", c) forAllChildren(c, waDebug) c_fprintf(file, "end\n") - gch.dumpHeapFile = nil + gch.pDumpHeapFile = nil proc GC_dumpHeap() = var f: File diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 7a1b88c84..513ede173 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -120,6 +120,7 @@ when allowForeignThreadGc: ## switches are used if not localGcInitialized: localGcInitialized = true + initAllocator() var stackTop {.volatile.}: pointer setStackBottom(addr(stackTop)) initGC() diff --git a/lib/system/jssys.nim b/lib/system/jssys.nim index 9c8a18bfe..39ce1183b 100644 --- a/lib/system/jssys.nim +++ b/lib/system/jssys.nim @@ -7,11 +7,6 @@ # distribution, for details about the copyright. # -when defined(nodejs): - proc alert*(s: cstring) {.importc: "console.log", nodecl.} -else: - proc alert*(s: cstring) {.importc, nodecl.} - proc log*(s: cstring) {.importc: "console.log", varargs, nodecl.} type @@ -52,11 +47,30 @@ proc nimCharToStr(x: char): string {.compilerproc.} = result = newString(1) result[0] = x +proc isNimException(): bool {.asmNoStackFrame.} = + when defined(nimphp): + asm "return isset(`lastJSError`['m_type']);" + else: + asm "return `lastJSError`.m_type;" + +proc getCurrentException*(): ref Exception = + if isNimException(): result = cast[ref Exception](lastJSError) + proc getCurrentExceptionMsg*(): string = if lastJSError != nil: - return $lastJSError.message - else: - return "" + if isNimException(): + return cast[Exception](lastJSError).msg + else: + when not defined(nimphp): + var msg: cstring + {.emit: """ + if (`lastJSError`.message !== undefined) { + `msg` = `lastJSError`.message; + } + """.} + if not msg.isNil: + return $msg + return "" proc auxWriteStackTrace(f: PCallFrame): string = type @@ -91,54 +105,56 @@ proc auxWriteStackTrace(f: PCallFrame): string = proc rawWriteStackTrace(): string = if framePtr != nil: result = "Traceback (most recent call last)\n" & auxWriteStackTrace(framePtr) - framePtr = nil - elif lastJSError != nil: - result = $lastJSError.stack else: result = "No stack traceback available\n" proc getStackTrace*(): string = rawWriteStackTrace() +proc getStackTrace*(e: ref Exception): string = e.trace proc unhandledException(e: ref Exception) {. compilerproc, asmNoStackFrame.} = - when NimStackTrace: - var buf = rawWriteStackTrace() + var buf = "" + if e.msg != nil and e.msg[0] != '\0': + add(buf, "Error: unhandled exception: ") + add(buf, e.msg) else: - var buf = "" - if e.msg != nil and e.msg[0] != '\0': - add(buf, "Error: unhandled exception: ") - add(buf, e.msg) - else: - add(buf, "Error: unhandled exception") - add(buf, " [") - add(buf, e.name) - add(buf, "]\n") - alert(buf) + add(buf, "Error: unhandled exception") + add(buf, " [") + add(buf, e.name) + add(buf, "]\n") + when NimStackTrace: + add(buf, rawWriteStackTrace()) + let cbuf : cstring = buf + framePtr = nil + {.emit: """ + if (typeof(Error) !== "undefined") { + throw new Error(`cbuf`); + } + else { + throw `cbuf`; + } + """.} proc raiseException(e: ref Exception, ename: cstring) {. compilerproc, asmNoStackFrame.} = e.name = ename - when not defined(noUnhandledHandler): - if excHandler == 0: - unhandledException(e) + if excHandler == 0: + unhandledException(e) when defined(nimphp): asm """throw new Exception($`e`["message"]);""" else: + when NimStackTrace: + e.trace = rawWriteStackTrace() asm "throw `e`;" proc reraiseException() {.compilerproc, asmNoStackFrame.} = if lastJSError == nil: raise newException(ReraiseError, "no exception to reraise") else: - when not defined(noUnhandledHandler): - if excHandler == 0: - var isNimException: bool - when defined(nimphp): - asm "`isNimException` = isset(`lastJSError`['m_type']);" - else: - asm "`isNimException` = lastJSError.m_type;" - if isNimException: - unhandledException(cast[ref Exception](lastJSError)) + if excHandler == 0: + if isNimException(): + unhandledException(cast[ref Exception](lastJSError)) + asm "throw lastJSError;" proc raiseOverflow {.exportc: "raiseOverflow", noreturn.} = @@ -873,3 +889,10 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, start = 0): int { # evaluate sign number = number * sign result = i - start + +when defined(nodejs): + # Deprecated. Use `alert` defined in dom.nim + proc alert*(s: cstring) {.importc: "console.log", nodecl, deprecated.} +else: + # Deprecated. Use `alert` defined in dom.nim + proc alert*(s: cstring) {.importc, nodecl, deprecated.} diff --git a/lib/system/mmdisp.nim b/lib/system/mmdisp.nim index 186349152..63af49e35 100644 --- a/lib/system/mmdisp.nim +++ b/lib/system/mmdisp.nim @@ -21,7 +21,7 @@ const alwaysGC = defined(fulldebug) # collect after every memory # allocation (for debugging) leakDetector = false - overwriteFree = false + overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free trackAllocationSource = leakDetector cycleGC = true # (de)activate the cycle GC diff --git a/lib/system/nimscript.nim b/lib/system/nimscript.nim index fc6b8c99d..29466c34d 100644 --- a/lib/system/nimscript.nim +++ b/lib/system/nimscript.nim @@ -301,7 +301,7 @@ var author*: string ## Nimble support: The package's author. description*: string ## Nimble support: The package's description. license*: string ## Nimble support: The package's license. - srcdir*: string ## Nimble support: The package's source directory. + srcDir*: string ## Nimble support: The package's source directory. binDir*: string ## Nimble support: The package's binary directory. backend*: string ## Nimble support: The package's backend. diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index b07a362a0..316dd74d7 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -87,8 +87,6 @@ elif defined(posix): const MAP_ANONYMOUS = 0x1000 elif defined(solaris): const MAP_ANONYMOUS = 0x100 - elif defined(linux): - const MAP_ANONYMOUS = 0x20 else: var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index eb3d276e0..3a93221e0 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -110,6 +110,11 @@ proc copyStringRC1(src: NimString): NimString {.compilerRtl.} = result.len = src.len copyMem(addr(result.data), addr(src.data), src.len + 1) +proc copyDeepString(src: NimString): NimString {.inline.} = + if src != nil: + result = rawNewStringNoInit(src.len) + result.len = src.len + copyMem(addr(result.data), addr(src.data), src.len + 1) proc hashString(s: string): int {.compilerproc.} = # the compiler needs exactly the same hash function! @@ -292,7 +297,7 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = buf[n+2] = '\0' # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' are produced. # We want to get rid of these here: - if buf[n-1] == 'N': + if buf[n-1] in {'n', 'N'}: result = "nan" elif buf[n-1] == 'F': if buf[0] == '-': diff --git a/lib/system/threads.nim b/lib/system/threads.nim index 583e3ae86..6f5bb38b1 100644 --- a/lib/system/threads.nim +++ b/lib/system/threads.nim @@ -331,7 +331,9 @@ else: when TArg is void: thrd.dataFn() else: - thrd.dataFn(thrd.data) + var x: TArg + deepCopy(x, thrd.data) + thrd.dataFn(x) proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = when defined(boehmgc): @@ -354,6 +356,8 @@ proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) = template threadProcWrapperBody(closure: expr) {.immediate.} = when declared(globalsSlot): threadVarSetValue(globalsSlot, closure) + when declared(initAllocator): + initAllocator() var thrd = cast[ptr Thread[TArg]](closure) threadProcWrapStackFrame(thrd) # Since an unhandled exception terminates the whole process (!), there is @@ -419,7 +423,7 @@ when false: when hostOS == "windows": proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread.}, + tp: proc (arg: TArg) {.thread, nimcall.}, param: TArg) = ## creates a new thread `t` and starts its execution. Entry point is the ## proc `tp`. `param` is passed to `tp`. `TArg` can be ``void`` if you @@ -441,7 +445,7 @@ when hostOS == "windows": else: proc createThread*[TArg](t: var Thread[TArg], - tp: proc (arg: TArg) {.thread.}, + tp: proc (arg: TArg) {.thread, nimcall.}, param: TArg) = ## creates a new thread `t` and starts its execution. Entry point is the ## proc `tp`. `param` is passed to `tp`. `TArg` can be ``void`` if you @@ -464,7 +468,7 @@ else: cpusetIncl(cpu.cint, s) setAffinity(t.sys, sizeof(s), s) -proc createThread*(t: var Thread[void], tp: proc () {.thread.}) = +proc createThread*(t: var Thread[void], tp: proc () {.thread, nimcall.}) = createThread[void](t, tp) proc threadId*[TArg](t: var Thread[TArg]): ThreadId[TArg] {.inline.} = diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 19c9815d2..29b955c46 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -9,7 +9,7 @@ include "system/inclrtl" -import os, oids, tables, strutils, macros, times, heapqueue +import os, oids, tables, strutils, times, heapqueue import nativesockets, net, queues @@ -130,275 +130,7 @@ export Port, SocketFlag # TODO: Check if yielded future is nil and throw a more meaningful exception -# -- Futures - -type - FutureBase* = ref object of RootObj ## Untyped future. - cb: proc () {.closure,gcsafe.} - finished: bool - error*: ref Exception ## Stored exception - errorStackTrace*: string - when not defined(release): - stackTrace: string ## For debugging purposes only. - id: int - fromProc: string - - Future*[T] = ref object of FutureBase ## Typed future. - value: T ## Stored value - - FutureVar*[T] = distinct Future[T] - - FutureError* = object of Exception - cause*: FutureBase - -{.deprecated: [PFutureBase: FutureBase, PFuture: Future].} - -when not defined(release): - var currentID = 0 - -proc callSoon*(cbproc: proc ()) {.gcsafe.} - -proc newFuture*[T](fromProc: string = "unspecified"): Future[T] = - ## Creates a new future. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - new(result) - result.finished = false - when not defined(release): - result.stackTrace = getStackTrace() - result.id = currentID - result.fromProc = fromProc - currentID.inc() - -proc newFutureVar*[T](fromProc = "unspecified"): FutureVar[T] = - ## Create a new ``FutureVar``. This Future type is ideally suited for - ## situations where you want to avoid unnecessary allocations of Futures. - ## - ## Specifying ``fromProc``, which is a string specifying the name of the proc - ## that this future belongs to, is a good habit as it helps with debugging. - result = FutureVar[T](newFuture[T](fromProc)) - -proc clean*[T](future: FutureVar[T]) = - ## Resets the ``finished`` status of ``future``. - Future[T](future).finished = false - Future[T](future).error = nil - -proc checkFinished[T](future: Future[T]) = - ## Checks whether `future` is finished. If it is then raises a - ## ``FutureError``. - when not defined(release): - if future.finished: - var msg = "" - msg.add("An attempt was made to complete a Future more than once. ") - msg.add("Details:") - msg.add("\n Future ID: " & $future.id) - msg.add("\n Created in proc: " & future.fromProc) - msg.add("\n Stack trace to moment of creation:") - msg.add("\n" & indent(future.stackTrace.strip(), 4)) - when T is string: - msg.add("\n Contents (string): ") - msg.add("\n" & indent(future.value.repr, 4)) - msg.add("\n Stack trace to moment of secondary completion:") - msg.add("\n" & indent(getStackTrace().strip(), 4)) - var err = newException(FutureError, msg) - err.cause = future - raise err - -proc complete*[T](future: Future[T], val: T) = - ## Completes ``future`` with value ``val``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.value = val - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*(future: Future[void]) = - ## Completes a void ``future``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - assert(future.error == nil) - future.finished = true - if future.cb != nil: - future.cb() - -proc complete*[T](future: FutureVar[T]) = - ## Completes a ``FutureVar``. - template fut: expr = Future[T](future) - checkFinished(fut) - assert(fut.error == nil) - fut.finished = true - if fut.cb != nil: - fut.cb() - -proc fail*[T](future: Future[T], error: ref Exception) = - ## Completes ``future`` with ``error``. - #assert(not future.finished, "Future already finished, cannot finish twice.") - checkFinished(future) - future.finished = true - future.error = error - future.errorStackTrace = - if getStackTrace(error) == "": getStackTrace() else: getStackTrace(error) - if future.cb != nil: - future.cb() - else: - # This is to prevent exceptions from being silently ignored when a future - # is discarded. - # TODO: This may turn out to be a bad idea. - # Turns out this is a bad idea. - #raise error - discard - -proc `callback=`*(future: FutureBase, cb: proc () {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - ## - ## **Note**: You most likely want the other ``callback`` setter which - ## passes ``future`` as a param to the callback. - future.cb = cb - if future.finished: - callSoon(future.cb) - -proc `callback=`*[T](future: Future[T], - cb: proc (future: Future[T]) {.closure,gcsafe.}) = - ## Sets the callback proc to be called when the future completes. - ## - ## If future has already completed then ``cb`` will be called immediately. - future.callback = proc () = cb(future) - -proc injectStacktrace[T](future: Future[T]) = - # TODO: Come up with something better. - when not defined(release): - var msg = "" - msg.add("\n " & future.fromProc & "'s lead up to read of failed Future:") - - if not future.errorStackTrace.isNil and future.errorStackTrace != "": - msg.add("\n" & indent(future.errorStackTrace.strip(), 4)) - else: - msg.add("\n Empty or nil stack trace.") - future.error.msg.add(msg) - -proc read*[T](future: Future[T]): T = - ## Retrieves the value of ``future``. Future must be finished otherwise - ## this function will fail with a ``ValueError`` exception. - ## - ## If the result of the future is an error then that error will be raised. - if future.finished: - if future.error != nil: - injectStacktrace(future) - raise future.error - when T isnot void: - return future.value - else: - # TODO: Make a custom exception type for this? - raise newException(ValueError, "Future still in progress.") - -proc readError*[T](future: Future[T]): ref Exception = - ## Retrieves the exception stored in ``future``. - ## - ## An ``ValueError`` exception will be thrown if no exception exists - ## in the specified Future. - if future.error != nil: return future.error - else: - raise newException(ValueError, "No error in future.") - -proc mget*[T](future: FutureVar[T]): var T = - ## Returns a mutable value stored in ``future``. - ## - ## Unlike ``read``, this function will not raise an exception if the - ## Future has not been finished. - result = Future[T](future).value - -proc finished*[T](future: Future[T]): bool = - ## Determines whether ``future`` has completed. - ## - ## ``True`` may indicate an error or a value. Use ``failed`` to distinguish. - future.finished - -proc failed*(future: FutureBase): bool = - ## Determines whether ``future`` completed with an error. - return future.error != nil - -proc asyncCheck*[T](future: Future[T]) = - ## Sets a callback on ``future`` which raises an exception if the future - ## finished with an error. - ## - ## This should be used instead of ``discard`` to discard void futures. - future.callback = - proc () = - if future.failed: - injectStacktrace(future) - raise future.error - -proc `and`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once both ``fut1`` and ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`and`") - fut1.callback = - proc () = - if fut2.finished: retFuture.complete() - fut2.callback = - proc () = - if fut1.finished: retFuture.complete() - return retFuture - -proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = - ## Returns a future which will complete once either ``fut1`` or ``fut2`` - ## complete. - var retFuture = newFuture[void]("asyncdispatch.`or`") - proc cb() = - if not retFuture.finished: retFuture.complete() - fut1.callback = cb - fut2.callback = cb - return retFuture - -proc all*[T](futs: varargs[Future[T]]): auto = - ## Returns a future which will complete once - ## all futures in ``futs`` complete. - ## - ## If the awaited futures are not ``Future[void]``, the returned future - ## will hold the values of all awaited futures in a sequence. - ## - ## If the awaited futures *are* ``Future[void]``, - ## this proc returns ``Future[void]``. - - when T is void: - var - retFuture = newFuture[void]("asyncdispatch.all") - completedFutures = 0 - - let totalFutures = len(futs) - - for fut in futs: - fut.callback = proc(f: Future[T]) = - inc(completedFutures) - - if completedFutures == totalFutures: - retFuture.complete() - - return retFuture - - else: - var - retFuture = newFuture[seq[T]]("asyncdispatch.all") - retValues = newSeq[T](len(futs)) - completedFutures = 0 - - for i, fut in futs: - proc setCallback(i: int) = - fut.callback = proc(f: Future[T]) = - retValues[i] = f.read() - inc(completedFutures) - - if completedFutures == len(retValues): - retFuture.complete(retValues) - - setCallback(i) - - return retFuture +include "../includes/asyncfutures" type PDispatcherBase = ref object of RootRef @@ -511,43 +243,44 @@ when defined(windows) or defined(nimdoc): if at == -1: winlean.INFINITE else: at.int32 - var lpNumberOfBytesTransferred: Dword - var lpCompletionKey: ULONG_PTR - var customOverlapped: PCustomOverlapped - let res = getQueuedCompletionStatus(p.ioPort, - addr lpNumberOfBytesTransferred, addr lpCompletionKey, - cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool - - # http://stackoverflow.com/a/12277264/492186 - # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html - if res: - # This is useful for ensuring the reliability of the overlapped struct. - assert customOverlapped.data.fd == lpCompletionKey.AsyncFD - - customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, OSErrorCode(-1)) - - # If cell.data != nil, then system.protect(rawEnv(cb)) was called, - # so we need to dispose our `cb` environment, because it is not needed - # anymore. - if customOverlapped.data.cell.data != nil: - system.dispose(customOverlapped.data.cell) - - GC_unref(customOverlapped) - else: - let errCode = osLastError() - if customOverlapped != nil: + if p.handles.len != 0: + var lpNumberOfBytesTransferred: Dword + var lpCompletionKey: ULONG_PTR + var customOverlapped: PCustomOverlapped + let res = getQueuedCompletionStatus(p.ioPort, + addr lpNumberOfBytesTransferred, addr lpCompletionKey, + cast[ptr POVERLAPPED](addr customOverlapped), llTimeout).bool + + # http://stackoverflow.com/a/12277264/492186 + # TODO: http://www.serverframework.com/handling-multiple-pending-socket-read-and-write-operations.html + if res: + # This is useful for ensuring the reliability of the overlapped struct. assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, - lpNumberOfBytesTransferred, errCode) + lpNumberOfBytesTransferred, OSErrorCode(-1)) + + # If cell.data != nil, then system.protect(rawEnv(cb)) was called, + # so we need to dispose our `cb` environment, because it is not needed + # anymore. if customOverlapped.data.cell.data != nil: system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) else: - if errCode.int32 == WAIT_TIMEOUT: - # Timed out - discard - else: raiseOSError(errCode) + let errCode = osLastError() + if customOverlapped != nil: + assert customOverlapped.data.fd == lpCompletionKey.AsyncFD + customOverlapped.data.cb(customOverlapped.data.fd, + lpNumberOfBytesTransferred, errCode) + if customOverlapped.data.cell.data != nil: + system.dispose(customOverlapped.data.cell) + GC_unref(customOverlapped) + else: + if errCode.int32 == WAIT_TIMEOUT: + # Timed out + discard + else: raiseOSError(errCode) # Timer processing. processTimers(p) @@ -718,7 +451,7 @@ when defined(windows) or defined(nimdoc): retFuture.complete("") return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[int] = ## Reads **up to** ``size`` bytes from ``socket`` into ``buf``, which must ## at least be of that size. Returned future will complete once all the @@ -742,7 +475,7 @@ when defined(windows) or defined(nimdoc): #buf[] = '\0' var dataBuf: TWSABuf - dataBuf.buf = buf + dataBuf.buf = cast[cstring](buf) dataBuf.len = size.ULONG var bytesReceived: Dword @@ -753,10 +486,7 @@ when defined(windows) or defined(nimdoc): proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = if not retFuture.finished: if errcode == OSErrorCode(-1): - if bytesCount == 0 and dataBuf.buf[0] == '\0': - retFuture.complete(0) - else: - retFuture.complete(bytesCount) + retFuture.complete(bytesCount) else: if flags.isDisconnectionError(errcode): retFuture.complete(0) @@ -788,6 +518,51 @@ when defined(windows) or defined(nimdoc): retFuture.complete(bytesReceived) return retFuture + proc send*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): Future[void] = + ## Sends ``size`` bytes from ``buf`` to ``socket``. The returned future will complete once all + ## data has been sent. + ## **WARNING**: Use it with caution. If ``buf`` refers to GC'ed object, you must use GC_ref/GC_unref calls + ## to avoid early freeing of the buffer + verifyPresence(socket) + var retFuture = newFuture[void]("send") + + var dataBuf: TWSABuf + dataBuf.buf = cast[cstring](buf) + dataBuf.len = size.ULONG + + var bytesReceived, lowFlags: Dword + var ol = PCustomOverlapped() + GC_ref(ol) + ol.data = CompletionData(fd: socket, cb: + proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + if not retFuture.finished: + if errcode == OSErrorCode(-1): + retFuture.complete() + else: + if flags.isDisconnectionError(errcode): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(errcode))) + ) + + let ret = WSASend(socket.SocketHandle, addr dataBuf, 1, addr bytesReceived, + lowFlags, cast[POVERLAPPED](ol), nil) + if ret == -1: + let err = osLastError() + if err.int32 != ERROR_IO_PENDING: + GC_unref(ol) + if flags.isDisconnectionError(err): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) + else: + retFuture.complete() + # We don't deallocate ``ol`` here because even though this completed + # immediately poll will still be notified about its completion and it will + # free ``ol``. + return retFuture + proc send*(socket: AsyncFD, data: string, flags = {SocketFlag.SafeDisconn}): Future[void] = ## Sends ``data`` to ``socket``. The returned future will complete once all @@ -796,7 +571,9 @@ when defined(windows) or defined(nimdoc): var retFuture = newFuture[void]("send") var dataBuf: TWSABuf - dataBuf.buf = data # since this is not used in a callback, this is fine + dataBuf.buf = data + GC_ref(data) # we need to protect data until send operation is completed + # or failed. dataBuf.len = data.len.ULONG var bytesReceived, lowFlags: Dword @@ -804,6 +581,7 @@ when defined(windows) or defined(nimdoc): GC_ref(ol) ol.data = CompletionData(fd: socket, cb: proc (fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = + GC_unref(data) # if operation completed `data` must be released. if not retFuture.finished: if errcode == OSErrorCode(-1): retFuture.complete() @@ -820,6 +598,8 @@ when defined(windows) or defined(nimdoc): let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) + GC_unref(data) # if operation failed `data` must be released, because + # completion routine will not be called. if flags.isDisconnectionError(err): retFuture.complete() else: @@ -952,7 +732,7 @@ when defined(windows) or defined(nimdoc): let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) - template completeAccept(): stmt {.immediate, dirty.} = + template completeAccept() {.dirty.} = var listenSock = socket let setoptRet = setsockopt(clientSock, SOL_SOCKET, SO_UPDATE_ACCEPT_CONTEXT, addr listenSock, @@ -972,7 +752,7 @@ when defined(windows) or defined(nimdoc): client: clientSock.AsyncFD) ) - template failAccept(errcode): stmt = + template failAccept(errcode) = if flags.isDisconnectionError(errcode): var newAcceptFut = acceptAddr(socket, flags) newAcceptFut.callback = @@ -1158,7 +938,7 @@ when defined(windows) or defined(nimdoc): ## receiving notifies. registerWaitableEvent(FD_WRITE or FD_CONNECT or FD_CLOSE) - template registerWaitableHandle(p, hEvent, flags, pcd, handleCallback) = + template registerWaitableHandle(p, hEvent, flags, pcd, timeout, handleCallback) = let handleFD = AsyncFD(hEvent) pcd.ioPort = p.ioPort pcd.handleFd = handleFD @@ -1172,10 +952,10 @@ when defined(windows) or defined(nimdoc): pcd.ovl = ol if not registerWaitForSingleObject(addr(pcd.waitFd), hEvent, cast[WAITORTIMERCALLBACK](waitableCallback), - cast[pointer](pcd), INFINITE, flags): + cast[pointer](pcd), timeout.Dword, flags): GC_unref(ol) deallocShared(cast[pointer](pcd)) - discard wsaCloseEvent(hEvent) + discard closeHandle(hEvent) raiseOSError(osLastError()) p.handles.incl(handleFD) @@ -1207,7 +987,7 @@ when defined(windows) or defined(nimdoc): deallocShared(cast[pointer](pcd)) p.handles.excl(fd) - registerWaitableHandle(p, hEvent, flags, pcd, timercb) + registerWaitableHandle(p, hEvent, flags, pcd, timeout, timercb) proc addProcess*(pid: int, cb: Callback) = ## Registers callback ``cb`` to be called when process with pid ``pid`` @@ -1231,10 +1011,12 @@ when defined(windows) or defined(nimdoc): p.handles.excl(fd) discard cb(fd) - registerWaitableHandle(p, hProcess, flags, pcd, proccb) + registerWaitableHandle(p, hProcess, flags, pcd, INFINITE, proccb) proc newAsyncEvent*(): AsyncEvent = ## Creates new ``AsyncEvent`` object. + ## New ``AsyncEvent`` object is not automatically registered with + ## dispatcher like ``AsyncSocket``. var sa = SECURITY_ATTRIBUTES( nLength: sizeof(SECURITY_ATTRIBUTES).cint, bInheritHandle: 1 @@ -1243,14 +1025,15 @@ when defined(windows) or defined(nimdoc): if event == INVALID_HANDLE_VALUE: raiseOSError(osLastError()) result = cast[AsyncEvent](allocShared0(sizeof(AsyncEventImpl))) + result.hEvent = event proc setEvent*(ev: AsyncEvent) = ## Set event ``ev`` to signaled state. if setEvent(ev.hEvent) == 0: raiseOSError(osLastError()) - proc close*(ev: AsyncEvent) = - ## Closes event ``ev``. + proc unregister*(ev: AsyncEvent) = + ## Unregisters event ``ev``. if ev.hWaiter != 0: let p = getGlobalDispatcher() if unregisterWait(ev.hWaiter) == 0: @@ -1258,7 +1041,12 @@ when defined(windows) or defined(nimdoc): if err.int32 != ERROR_IO_PENDING: raiseOSError(osLastError()) p.handles.excl(AsyncFD(ev.hEvent)) + ev.hWaiter = 0 + else: + raise newException(ValueError, "Event is not registered!") + proc close*(ev: AsyncEvent) = + ## Closes event ``ev``. if closeHandle(ev.hEvent) == 0: raiseOSError(osLastError()) deallocShared(cast[pointer](ev)) @@ -1276,15 +1064,12 @@ when defined(windows) or defined(nimdoc): proc eventcb(fd: AsyncFD, bytesCount: Dword, errcode: OSErrorCode) = if cb(fd): - if unregisterWait(pcd.waitFd) == 0: - let err = osLastError() - if err.int32 != ERROR_IO_PENDING: - raiseOSError(osLastError()) - ev.hWaiter = 0 + # we need this check to avoid exception, if `unregister(event)` was + # called in callback. + if ev.hWaiter != 0: unregister(ev) deallocShared(cast[pointer](pcd)) - p.handles.excl(fd) - registerWaitableHandle(p, hEvent, flags, pcd, eventcb) + registerWaitableHandle(p, hEvent, flags, pcd, INFINITE, eventcb) ev.hWaiter = pcd.waitFd initAll() @@ -1314,13 +1099,14 @@ else: readCB: Callback writeCB: Callback - AsyncEvent* = SelectEvent + AsyncEvent* = distinct SelectEvent PDispatcher* = ref object of PDispatcherBase selector: Selector[AsyncData] {.deprecated: [TAsyncFD: AsyncFD, TCallback: Callback].} proc `==`*(x, y: AsyncFD): bool {.borrow.} + proc `==`*(x, y: AsyncEvent): bool {.borrow.} proc newDispatcher*(): PDispatcher = new result @@ -1363,24 +1149,30 @@ else: proc unregister*(fd: AsyncFD) = getGlobalDispatcher().selector.unregister(fd.SocketHandle) - # proc unregister*(ev: AsyncEvent) = - # getGlobalDispatcher().selector.unregister(SelectEvent(ev)) + proc unregister*(ev: AsyncEvent) = + getGlobalDispatcher().selector.unregister(SelectEvent(ev)) proc addRead*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() + var newEvents = {Event.Read} withData(p.selector, fd.SocketHandle, adata) do: adata.readCB = cb + if adata.writeCB != nil: + newEvents.incl(Event.Write) do: raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, {Event.Read}) + p.selector.updateHandle(fd.SocketHandle, newEvents) proc addWrite*(fd: AsyncFD, cb: Callback) = let p = getGlobalDispatcher() + var newEvents = {Event.Write} withData(p.selector, fd.SocketHandle, adata) do: adata.writeCB = cb + if adata.readCB != nil: + newEvents.incl(Event.Read) do: raise newException(ValueError, "File descriptor not registered.") - p.selector.updateHandle(fd.SocketHandle, {Event.Write}) + p.selector.updateHandle(fd.SocketHandle, newEvents) proc poll*(timeout = 500) = var keys: array[64, ReadyKey[AsyncData]] @@ -1398,7 +1190,7 @@ else: var count = p.selector.selectInto(p.adjustedTimeout(timeout), keys) var i = 0 while i < count: - var update = false + var custom = false var fd = keys[i].fd.SocketHandle let events = keys[i].events @@ -1409,7 +1201,6 @@ else: p.selector.withData(fd, adata) do: if adata.readCB == cb: adata.readCB = nil - update = true if Event.Write in events: let cb = keys[i].data.writeCB @@ -1418,24 +1209,29 @@ else: p.selector.withData(fd, adata) do: if adata.writeCB == cb: adata.writeCB = nil - update = true when supportedPlatform: if (customSet * events) != {}: let cb = keys[i].data.readCB doAssert(cb != nil) + custom = true if cb(fd.AsyncFD): p.selector.withData(fd, adata) do: if adata.readCB == cb: adata.readCB = nil p.selector.unregister(fd) - if update: + # because state `data` can be modified in callback we need to update + # descriptor events with currently registered callbacks. + if not custom: + var update = false var newEvents: set[Event] = {} p.selector.withData(fd, adata) do: if adata.readCB != nil: incl(newEvents, Event.Read) if adata.writeCB != nil: incl(newEvents, Event.Write) - p.selector.updateHandle(fd, newEvents) + update = true + if update: + p.selector.updateHandle(fd, newEvents) inc(i) # Timer processing. @@ -1517,7 +1313,7 @@ else: addRead(socket, cb) return retFuture - proc recvInto*(socket: AsyncFD, buf: cstring, size: int, + proc recvInto*(socket: AsyncFD, buf: pointer, size: int, flags = {SocketFlag.SafeDisconn}): Future[int] = var retFuture = newFuture[int]("recvInto") @@ -1541,6 +1337,38 @@ else: addRead(socket, cb) return retFuture + proc send*(socket: AsyncFD, buf: pointer, size: int, + flags = {SocketFlag.SafeDisconn}): Future[void] = + var retFuture = newFuture[void]("send") + + var written = 0 + + proc cb(sock: AsyncFD): bool = + result = true + let netSize = size-written + var d = cast[cstring](buf) + let res = send(sock.SocketHandle, addr d[written], netSize.cint, + MSG_NOSIGNAL) + if res < 0: + let lastError = osLastError() + if lastError.int32 notin {EINTR, EWOULDBLOCK, EAGAIN}: + if flags.isDisconnectionError(lastError): + retFuture.complete() + else: + retFuture.fail(newException(OSError, osErrorMsg(lastError))) + else: + result = false # We still want this callback to be called. + else: + written.inc(res) + if res != netSize: + result = false # We still have data to send. + else: + retFuture.complete() + # TODO: The following causes crashes. + #if not cb(socket): + addWrite(socket, cb) + return retFuture + proc send*(socket: AsyncFD, data: string, flags = {SocketFlag.SafeDisconn}): Future[void] = var retFuture = newFuture[void]("send") @@ -1682,15 +1510,15 @@ else: proc newAsyncEvent*(): AsyncEvent = ## Creates new ``AsyncEvent``. - result = AsyncEvent(ioselectors.newSelectEvent()) + result = AsyncEvent(newSelectEvent()) proc setEvent*(ev: AsyncEvent) = ## Sets new ``AsyncEvent`` to signaled state. - ioselectors.setEvent(SelectEvent(ev)) + setEvent(SelectEvent(ev)) proc close*(ev: AsyncEvent) = ## Closes ``AsyncEvent`` - ioselectors.close(SelectEvent(ev)) + close(SelectEvent(ev)) proc addEvent*(ev: AsyncEvent, cb: Callback) = ## Start watching for event ``ev``, and call callback ``cb``, when @@ -1742,360 +1570,7 @@ proc accept*(socket: AsyncFD, return retFut # -- Await Macro - -proc skipUntilStmtList(node: NimNode): NimNode {.compileTime.} = - # Skips a nest of StmtList's. - result = node - if node[0].kind == nnkStmtList: - result = skipUntilStmtList(node[0]) - -proc skipStmtList(node: NimNode): NimNode {.compileTime.} = - result = node - if node[0].kind == nnkStmtList: - result = node[0] - -template createCb(retFutureSym, iteratorNameSym, - name: expr): stmt {.immediate.} = - var nameIterVar = iteratorNameSym - #{.push stackTrace: off.} - proc cb {.closure,gcsafe.} = - try: - if not nameIterVar.finished: - var next = nameIterVar() - if next == nil: - assert retFutureSym.finished, "Async procedure's (" & - name & ") return Future was not finished." - else: - next.callback = cb - except: - if retFutureSym.finished: - # Take a look at tasyncexceptions for the bug which this fixes. - # That test explains it better than I can here. - raise - else: - retFutureSym.fail(getCurrentException()) - cb() - #{.pop.} -proc generateExceptionCheck(futSym, - tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} = - if tryStmt.kind == nnkNilLit: - result = rootReceiver - else: - var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[] - let errorNode = newDotExpr(futSym, newIdentNode("error")) - for i in 1 .. <tryStmt.len: - let exceptBranch = tryStmt[i] - if exceptBranch[0].kind == nnkStmtList: - exceptionChecks.add((newIdentNode("true"), exceptBranch[0])) - else: - var exceptIdentCount = 0 - var ifCond: NimNode - for i in 0 .. <exceptBranch.len: - let child = exceptBranch[i] - if child.kind == nnkIdent: - let cond = infix(errorNode, "of", child) - if exceptIdentCount == 0: - ifCond = cond - else: - ifCond = infix(ifCond, "or", cond) - else: - break - exceptIdentCount.inc - - expectKind(exceptBranch[exceptIdentCount], nnkStmtList) - exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount])) - # -> -> else: raise futSym.error - exceptionChecks.add((newIdentNode("true"), - newNimNode(nnkRaiseStmt).add(errorNode))) - # Read the future if there is no error. - # -> else: futSym.read - let elseNode = newNimNode(nnkElse, fromNode) - elseNode.add newNimNode(nnkStmtList, fromNode) - elseNode[0].add rootReceiver - - let ifBody = newStmtList() - ifBody.add newCall(newIdentNode("setCurrentException"), errorNode) - ifBody.add newIfStmt(exceptionChecks) - ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit()) - - result = newIfStmt( - (newDotExpr(futSym, newIdentNode("failed")), ifBody) - ) - result.add elseNode - -template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver, - rootReceiver: expr, fromNode: NimNode) = - ## Params: - ## futureVarNode: The NimNode which is a symbol identifying the Future[T] - ## variable to yield. - ## fromNode: Used for better debug information (to give context). - ## valueReceiver: The node which defines an expression that retrieves the - ## future's value. - ## - ## rootReceiver: ??? TODO - # -> yield future<x> - result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode) - # -> future<x>.read - valueReceiver = newDotExpr(futureVarNode, newIdentNode("read")) - result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver, - fromNode) - -template createVar(result: var NimNode, futSymName: string, - asyncProc: NimNode, - valueReceiver, rootReceiver: expr, - fromNode: NimNode) = - result = newNimNode(nnkStmtList, fromNode) - var futSym = genSym(nskVar, "future") - result.add newVarStmt(futSym, asyncProc) # -> var future<x> = y - useVar(result, futSym, valueReceiver, rootReceiver, fromNode) - -proc processBody(node, retFutureSym: NimNode, - subTypeIsVoid: bool, - tryStmt: NimNode): NimNode {.compileTime.} = - #echo(node.treeRepr) - result = node - case node.kind - of nnkReturnStmt: - result = newNimNode(nnkStmtList, node) - if node[0].kind == nnkEmpty: - if not subTypeIsVoid: - result.add newCall(newIdentNode("complete"), retFutureSym, - newIdentNode("result")) - else: - result.add newCall(newIdentNode("complete"), retFutureSym) - else: - result.add newCall(newIdentNode("complete"), retFutureSym, - node[0].processBody(retFutureSym, subTypeIsVoid, tryStmt)) - - result.add newNimNode(nnkReturnStmt, node).add(newNilLit()) - return # Don't process the children of this return stmt - of nnkCommand, nnkCall: - if node[0].kind == nnkIdent and node[0].ident == !"await": - case node[1].kind - of nnkIdent, nnkInfix, nnkDotExpr: - # await x - # await x or y - result = newNimNode(nnkYieldStmt, node).add(node[1]) # -> yield x - of nnkCall, nnkCommand: - # await foo(p, x) - # await foo p, x - var futureValue: NimNode - result.createVar("future" & $node[1][0].toStrLit, node[1], futureValue, - futureValue, node) - else: - error("Invalid node kind in 'await', got: " & $node[1].kind) - elif node.len > 1 and node[1].kind == nnkCommand and - node[1][0].kind == nnkIdent and node[1][0].ident == !"await": - # foo await x - var newCommand = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newCommand[1], - newCommand, node) - - of nnkVarSection, nnkLetSection: - case node[0][2].kind - of nnkCommand: - if node[0][2][0].kind == nnkIdent and node[0][2][0].ident == !"await": - # var x = await y - var newVarSection = node # TODO: Should this use copyNimNode? - result.createVar("future" & $node[0][0].ident, node[0][2][1], - newVarSection[0][2], newVarSection, node) - else: discard - of nnkAsgn: - case node[1].kind - of nnkCommand: - if node[1][0].ident == !"await": - # x = await y - var newAsgn = node - result.createVar("future" & $node[0].toStrLit, node[1][1], newAsgn[1], newAsgn, node) - else: discard - of nnkDiscardStmt: - # discard await x - if node[0].kind == nnkCommand and node[0][0].kind == nnkIdent and - node[0][0].ident == !"await": - var newDiscard = node - result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1], - newDiscard[0], newDiscard, node) - of nnkTryStmt: - # try: await x; except: ... - result = newNimNode(nnkStmtList, node) - template wrapInTry(n, tryBody: expr) = - var temp = n - n[0] = tryBody - tryBody = temp - - # Transform ``except`` body. - # TODO: Could we perform some ``await`` transformation here to get it - # working in ``except``? - tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid, nil) - - proc processForTry(n: NimNode, i: var int, - res: NimNode): bool {.compileTime.} = - ## Transforms the body of the tryStmt. Does not transform the - ## body in ``except``. - ## Returns true if the tryStmt node was transformed into an ifStmt. - result = false - var skipped = n.skipStmtList() - while i < skipped.len: - var processed = processBody(skipped[i], retFutureSym, - subTypeIsVoid, n) - - # Check if we transformed the node into an exception check. - # This suggests skipped[i] contains ``await``. - if processed.kind != skipped[i].kind or processed.len != skipped[i].len: - processed = processed.skipUntilStmtList() - expectKind(processed, nnkStmtList) - expectKind(processed[2][1], nnkElse) - i.inc - - if not processForTry(n, i, processed[2][1][0]): - # We need to wrap the nnkElse nodes back into a tryStmt. - # As they are executed if an exception does not happen - # inside the awaited future. - # The following code will wrap the nodes inside the - # original tryStmt. - wrapInTry(n, processed[2][1][0]) - - res.add processed - result = true - else: - res.add skipped[i] - i.inc - var i = 0 - if not processForTry(node, i, result): - # If the tryStmt hasn't been transformed we can just put the body - # back into it. - wrapInTry(node, result) - return - else: discard - - for i in 0 .. <result.len: - result[i] = processBody(result[i], retFutureSym, subTypeIsVoid, nil) - -proc getName(node: NimNode): string {.compileTime.} = - case node.kind - of nnkPostfix: - return $node[1].ident - of nnkIdent: - return $node.ident - of nnkEmpty: - return "anonymous" - else: - error("Unknown name.") - -proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} = - ## This macro transforms a single procedure into a closure iterator. - ## The ``async`` macro supports a stmtList holding multiple async procedures. - if prc.kind notin {nnkProcDef, nnkLambda}: - error("Cannot transform this node kind into an async proc." & - " Proc definition or lambda node expected.") - - hint("Processing " & prc[0].getName & " as an async proc.") - - let returnType = prc[3][0] - var baseType: NimNode - # Verify that the return type is a Future[T] - if returnType.kind == nnkBracketExpr: - let fut = repr(returnType[0]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[1] - elif returnType.kind in nnkCallKinds and $returnType[0] == "[]": - let fut = repr(returnType[1]) - if fut != "Future": - error("Expected return type of 'Future' got '" & fut & "'") - baseType = returnType[2] - elif returnType.kind == nnkEmpty: - baseType = returnType - else: - error("Expected return type of 'Future' got '" & repr(returnType) & "'") - - let subtypeIsVoid = returnType.kind == nnkEmpty or - (baseType.kind == nnkIdent and returnType[1].ident == !"void") - - var outerProcBody = newNimNode(nnkStmtList, prc[6]) - - # -> var retFuture = newFuture[T]() - var retFutureSym = genSym(nskVar, "retFuture") - var subRetType = - if returnType.kind == nnkEmpty: newIdentNode("void") - else: baseType - outerProcBody.add( - newVarStmt(retFutureSym, - newCall( - newNimNode(nnkBracketExpr, prc[6]).add( - newIdentNode(!"newFuture"), # TODO: Strange bug here? Remove the `!`. - subRetType), - newLit(prc[0].getName)))) # Get type from return type of this proc - - # -> iterator nameIter(): FutureBase {.closure.} = - # -> {.push warning[resultshadowed]: off.} - # -> var result: T - # -> {.pop.} - # -> <proc_body> - # -> complete(retFuture, result) - var iteratorNameSym = genSym(nskIterator, $prc[0].getName & "Iter") - var procBody = prc[6].processBody(retFutureSym, subtypeIsVoid, nil) - if not subtypeIsVoid: - procBody.insert(0, newNimNode(nnkPragma).add(newIdentNode("push"), - newNimNode(nnkExprColonExpr).add(newNimNode(nnkBracketExpr).add( - newIdentNode("warning"), newIdentNode("resultshadowed")), - newIdentNode("off")))) # -> {.push warning[resultshadowed]: off.} - - procBody.insert(1, newNimNode(nnkVarSection, prc[6]).add( - newIdentDefs(newIdentNode("result"), baseType))) # -> var result: T - - procBody.insert(2, newNimNode(nnkPragma).add( - newIdentNode("pop"))) # -> {.pop.}) - - procBody.add( - newCall(newIdentNode("complete"), - retFutureSym, newIdentNode("result"))) # -> complete(retFuture, result) - else: - # -> complete(retFuture) - procBody.add(newCall(newIdentNode("complete"), retFutureSym)) - - var closureIterator = newProc(iteratorNameSym, [newIdentNode("FutureBase")], - procBody, nnkIteratorDef) - closureIterator[4] = newNimNode(nnkPragma, prc[6]).add(newIdentNode("closure")) - outerProcBody.add(closureIterator) - - # -> createCb(retFuture) - #var cbName = newIdentNode("cb") - var procCb = newCall(bindSym"createCb", retFutureSym, iteratorNameSym, - newStrLitNode(prc[0].getName)) - outerProcBody.add procCb - - # -> return retFuture - outerProcBody.add newNimNode(nnkReturnStmt, prc[6][prc[6].len-1]).add(retFutureSym) - - result = prc - - # Remove the 'async' pragma. - for i in 0 .. <result[4].len: - if result[4][i].kind == nnkIdent and result[4][i].ident == !"async": - result[4].del(i) - result[4] = newEmptyNode() - if subtypeIsVoid: - # Add discardable pragma. - if returnType.kind == nnkEmpty: - # Add Future[void] - result[3][0] = parseExpr("Future[void]") - - result[6] = outerProcBody - - #echo(treeRepr(result)) - #if prc[0].getName == "testInfix": - # echo(toStrLit(result)) - -macro async*(prc: stmt): stmt {.immediate.} = - ## Macro which processes async procedures into the appropriate - ## iterators and yield statements. - if prc.kind == nnkStmtList: - for oneProc in prc: - result = newStmtList() - result.add asyncSingleProc(oneProc) - else: - result = asyncSingleProc(prc) +include asyncmacro proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## Reads a line of data from ``socket``. Returned future will complete once diff --git a/lib/windows/registry.nim b/lib/windows/registry.nim new file mode 100644 index 000000000..06f84c881 --- /dev/null +++ b/lib/windows/registry.nim @@ -0,0 +1,77 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module is experimental and its interface may change. + +import winlean, os + +type + HKEY* = uint + +const + HKEY_LOCAL_MACHINE* = HKEY(0x80000002u) + HKEY_CURRENT_USER* = HKEY(2147483649) + + RRF_RT_ANY = 0x0000ffff + KEY_WOW64_64KEY = 0x0100 + KEY_WOW64_32KEY = 0x0200 + KEY_READ = 0x00020019 + REG_SZ = 1 + +proc regOpenKeyEx(hKey: HKEY, lpSubKey: WideCString, ulOptions: int32, + samDesired: int32, + phkResult: var HKEY): int32 {. + importc: "RegOpenKeyExW", dynlib: "Advapi32.dll", stdcall.} + +proc regCloseKey(hkey: HKEY): int32 {. + importc: "RegCloseKey", dynlib: "Advapi32.dll", stdcall.} + +proc regGetValue(key: HKEY, lpSubKey, lpValue: WideCString; + dwFlags: int32 = RRF_RT_ANY, pdwType: ptr int32, + pvData: pointer, + pcbData: ptr int32): int32 {. + importc: "RegGetValueW", dynlib: "Advapi32.dll", stdcall.} + +template call(f) = + let err = f + if err != 0: + raiseOSError(err.OSErrorCode, astToStr(f)) + +proc getUnicodeValue*(path, key: string; handle: HKEY): string = + let hh = newWideCString path + let kk = newWideCString key + var bufsize: int32 + # try a couple of different flag settings: + var flags: int32 = RRF_RT_ANY + let err = regGetValue(handle, hh, kk, flags, nil, nil, addr bufsize) + if err != 0: + var newHandle: HKEY + call regOpenKeyEx(handle, hh, 0, KEY_READ or KEY_WOW64_64KEY, newHandle) + call regGetValue(newHandle, nil, kk, flags, nil, nil, addr bufsize) + var res = newWideCString("", bufsize) + call regGetValue(newHandle, nil, kk, flags, nil, cast[pointer](res), + addr bufsize) + result = res $ bufsize + call regCloseKey(newHandle) + else: + var res = newWideCString("", bufsize) + call regGetValue(handle, hh, kk, flags, nil, cast[pointer](res), + addr bufsize) + result = res $ bufsize + +proc regSetValue(key: HKEY, lpSubKey, lpValueName: WideCString, + dwType: int32; lpData: WideCString; cbData: int32): int32 {. + importc: "RegSetKeyValueW", dynlib: "Advapi32.dll", stdcall.} + +proc setUnicodeValue*(path, key, val: string; handle: HKey) = + let hh = newWideCString path + let kk = newWideCString key + let vv = newWideCString val + call regSetValue(handle, hh, kk, REG_SZ, vv, (vv.len.int32+1)*2) + diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index b766ead9c..2441c7267 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -337,11 +337,10 @@ when useWinUnicode: stdcall, dynlib: "kernel32", importc: "SetFileAttributesW".} proc copyFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: cint): cint {. + bFailIfExists: WINBOOL): WINBOOL {. importc: "CopyFileW", stdcall, dynlib: "kernel32".} - proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString, - bFailIfExists: cint): cint {. + proc moveFileW*(lpExistingFileName, lpNewFileName: WideCString): WINBOOL {. importc: "MoveFileW", stdcall, dynlib: "kernel32".} proc getEnvironmentStringsW*(): WideCString {. @@ -368,8 +367,7 @@ else: bFailIfExists: cint): cint {. importc: "CopyFileA", stdcall, dynlib: "kernel32".} - proc moveFileA*(lpExistingFileName, lpNewFileName: cstring, - bFailIfExists: cint): cint {. + proc moveFileA*(lpExistingFileName, lpNewFileName: cstring): WINBOOL {. importc: "MoveFileA", stdcall, dynlib: "kernel32".} proc getEnvironmentStringsA*(): cstring {. @@ -1035,4 +1033,4 @@ when defined(useWinAnsi): else: proc readConsoleInput*(hConsoleInput: Handle, lpBuffer: pointer, nLength: cint, lpNumberOfEventsRead: ptr cint): cint - {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} \ No newline at end of file + {.stdcall, dynlib: "kernel32", importc: "ReadConsoleInputW".} diff --git a/lib/wrappers/linenoise/clinenoise.c b/lib/wrappers/linenoise/clinenoise.c index a03e6f379..c76fc6586 100644 --- a/lib/wrappers/linenoise/clinenoise.c +++ b/lib/wrappers/linenoise/clinenoise.c @@ -116,7 +116,9 @@ #include <sys/types.h> #include <sys/ioctl.h> #include <unistd.h> -#include "clinenoise.h" +#ifndef __LINENOISE_H +# include "clinenoise.h" +#endif #define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 #define LINENOISE_MAX_LINE 4096 diff --git a/lib/wrappers/linenoise/linenoise.nim b/lib/wrappers/linenoise/linenoise.nim index b7004c6e2..4ac7bf4a8 100644 --- a/lib/wrappers/linenoise/linenoise.nim +++ b/lib/wrappers/linenoise/linenoise.nim @@ -14,7 +14,8 @@ type CompletionCallback* = proc (a2: cstring; a3: ptr Completions) {.cdecl.} -{.compile: "clinenoise.c".} +{.emit: staticRead"clinenoise.h".} +{.emit: staticRead"clinenoise.c".} proc setCompletionCallback*(a2: ptr CompletionCallback) {. importc: "linenoiseSetCompletionCallback".} diff --git a/lib/wrappers/mysql.nim b/lib/wrappers/mysql.nim index af504864d..6dbed23b3 100644 --- a/lib/wrappers/mysql.nim +++ b/lib/wrappers/mysql.nim @@ -13,10 +13,10 @@ when defined(Unix): when defined(macosx): const - lib = "libmysqlclient.(15|16|17|18).dylib" + lib = "libmysqlclient.(15|16|17|18|19|20).dylib" else: const - lib = "libmysqlclient.so.(15|16|17|18)" + lib = "libmysqlclient.so.(15|16|17|18|19|20)" when defined(Windows): const lib = "libmysql.dll" diff --git a/readme.md b/readme.md index f56b054d6..36d1ab0f9 100644 --- a/readme.md +++ b/readme.md @@ -51,6 +51,7 @@ instead of ``build.sh``. The ``koch`` tool is the Nim build tool, more ``koch`` related options are documented in [doc/koch.rst](doc/koch.rst). + ## Nimble [Nimble](https://github.com/nim-lang/nimble) is Nim's package manager. For the source based installations where you added Nim's ``bin`` directory to your PATH @@ -60,21 +61,18 @@ the easiest way of installing Nimble is via: $ nim e install_nimble.nims ``` -**Warning:** If you install Nimble this way, you will not be able to use binary -Nimble packages or update Nimble easily. -The [Nimble readme](https://github.com/nim-lang/nimble#installation) -provides thorough instructions on how to install Nimble, so that this isn't a -problem. - ## Community [](https://webchat.freenode.net/?channels=nim) +[](https://gitter.im/nim-lang/Nim) [](http://forum.nim-lang.org) [](http://stackoverflow.com/questions/tagged/nim?sort=newest&pageSize=15) [](https://twitter.com/nim_lang) * The [forum](http://forum.nim-lang.org/) - the best place to ask questions and to discuss Nim. -* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - the best place to discuss +* [IRC (Freenode#nim)](https://webchat.freenode.net/?channels=nim) - a place to discuss Nim in real-time, this is also where most development decision get made! +* [Gitter](https://gitter.im/nim-lang/Nim) allows to discuss Nim from your browser, one click to join. + There is a bridge between Gitter and IRC channels. * [Stackoverflow](http://stackoverflow.com/questions/tagged/nim) ## Contributing diff --git a/tests/array/troof2.nim b/tests/array/troof2.nim index d4c1a4982..e4b4f4b3c 100644 --- a/tests/array/troof2.nim +++ b/tests/array/troof2.nim @@ -2,8 +2,8 @@ discard """ errormsg: "invalid context for '^' as 'foo()' has side effects" line: "9" """ - -proc foo(): seq[int] = +# XXX This needs to be fixed properly! +proc foo(): seq[int] {.sideEffect.} = echo "ha" let f = foo()[^1] diff --git a/tests/async/config.nims b/tests/async/config.nims new file mode 100644 index 000000000..97c2e0aa4 --- /dev/null +++ b/tests/async/config.nims @@ -0,0 +1,2 @@ +when defined(upcoming): + patchFile("stdlib", "asyncdispatch", "$lib/upcoming/asyncdispatch") diff --git a/tests/async/tasyncdiscard.nim b/tests/async/tasyncdiscard.nim index e7c87ad42..64e6021c3 100644 --- a/tests/async/tasyncdiscard.nim +++ b/tests/async/tasyncdiscard.nim @@ -10,7 +10,7 @@ discard """ 6 ''' """ -import asyncio, asyncdispatch, asyncnet +import asyncdispatch, asyncnet proc main {.async.} = proc f: Future[int] {.async.} = diff --git a/tests/async/tasynciossl.nim b/tests/async/tasynciossl.nim deleted file mode 100644 index ba856760e..000000000 --- a/tests/async/tasynciossl.nim +++ /dev/null @@ -1,92 +0,0 @@ -discard """ - file: "tasynciossl.nim" - cmd: "nim $target --hints:on --define:ssl $options $file" - output: "20000" -""" -import sockets, asyncio, strutils, times - -var disp {.threadvar.}: PDispatcher -disp = newDispatcher() -var msgCount = 0 - -when defined(ssl): - var ctx = newContext(verifyMode = CVerifyNone, - certFile = "tests/testdata/mycert.pem", keyFile = "tests/testdata/mycert.pem") - - var ctx1 = newContext(verifyMode = CVerifyNone) - -const - swarmSize = 50 - messagesToSend = 100 - -proc swarmConnect(s: PAsyncSocket) = - #echo("Connected") - for i in 1..messagesToSend: - s.send("Message " & $i & "\c\L") - s.close() - -proc serverRead(s: PAsyncSocket) = - var line = "" - assert s.readLine(line) - if line != "": - #echo(line) - if line.startsWith("Message "): - msgCount.inc() - else: - assert(false) - else: - s.close() - -proc serverAccept(s: PAsyncSocket) = - var client: PAsyncSocket - new(client) - s.accept(client) - client.handleRead = serverRead - disp.register(client) - -proc launchSwarm(disp: var PDispatcher, port: TPort, count: int, - buffered = true, useSSL = false) = - for i in 1..count: - var client = asyncSocket() - when defined(ssl): - if useSSL: - ctx1.wrapSocket(client) - client.handleConnect = swarmConnect - disp.register(client) - client.connect("localhost", port) - -proc createSwarm(port: TPort, buffered = true, useSSL = false) = - var server = asyncSocket() - when defined(ssl): - if useSSL: - ctx.wrapSocket(server) - server.handleAccept = serverAccept - disp.register(server) - server.bindAddr(port) - server.listen() - disp.launchSwarm(port, swarmSize, buffered, useSSL) - -when defined(ssl): - const serverCount = 4 -else: - const serverCount = 2 - -createSwarm(TPort(10235)) -createSwarm(TPort(10236), false) - -when defined(ssl): - createSwarm(TPort(10237), true, true) - createSwarm(TPort(10238), false, true) - -var startTime = epochTime() -while true: - if epochTime() - startTime >= 300.0: - break - if not disp.poll(): break - if disp.len == serverCount: - # Only the servers are left in the dispatcher. All clients finished, - # we need to therefore break. - break - -assert msgCount == (swarmSize * messagesToSend) * serverCount -echo(msgCount) diff --git a/tests/async/tasyncsend4757.nim b/tests/async/tasyncsend4757.nim new file mode 100644 index 000000000..1066f38e5 --- /dev/null +++ b/tests/async/tasyncsend4757.nim @@ -0,0 +1,14 @@ +discard """ + file: "tasyncsend4754.nim" + output: "Finished" +""" + +import asyncdispatch + +proc f(): Future[void] {.async.} = + let s = newAsyncNativeSocket() + await s.connect("example.com", 80.Port) + await s.send("123") + echo "Finished" + +waitFor f() diff --git a/tests/async/tasyncssl.nim b/tests/async/tasyncssl.nim new file mode 100644 index 000000000..3dc131036 --- /dev/null +++ b/tests/async/tasyncssl.nim @@ -0,0 +1,66 @@ +discard """ + file: "tasyncssl.nim" + cmd: "nim $target --hints:on --define:ssl $options $file" + output: "500" +""" +import asyncdispatch, asyncnet, net, strutils, os + +when defined(ssl): + var msgCount = 0 + + const + swarmSize = 10 + messagesToSend = 50 + + var clientCount = 0 + + proc sendMessages(client: AsyncSocket) {.async.} = + for i in 0 .. <messagesToSend: + await send(client, "Message " & $i & "\c\L") + + proc launchSwarm(port: Port) {.async.} = + for i in 0 .. <swarmSize: + var sock = newAsyncSocket() + var clientContext = newContext(verifyMode = CVerifyNone) + clientContext.wrapSocket(sock) + await connect(sock, "localhost", port) + await sendMessages(sock) + close(sock) + + proc readMessages(client: AsyncSocket) {.async.} = + while true: + var line = await recvLine(client) + if line == "": + close(client) + inc(clientCount) + break + else: + if line.startswith("Message "): + inc(msgCount) + else: + doAssert false + + proc createServer(port: Port) {.async.} = + let serverContext = newContext(verifyMode = CVerifyNone, + certFile = "tests/testdata/mycert.pem", + keyFile = "tests/testdata/mycert.pem") + var server = newAsyncSocket() + serverContext.wrapSocket(server) + server.setSockOpt(OptReuseAddr, true) + bindAddr(server, port) + server.listen() + while true: + let client = await accept(server) + serverContext.wrapConnectedSocket(client, handshakeAsServer) + asyncCheck readMessages(client) + + asyncCheck createServer(Port(10335)) + asyncCheck launchSwarm(Port(10335)) + while true: + poll() + if clientCount == swarmSize: break + + assert msgCount == swarmSize * messagesToSend + echo msgCount + + diff --git a/tests/async/tasyncudp.nim b/tests/async/tasyncudp.nim deleted file mode 100644 index 57e2be85d..000000000 --- a/tests/async/tasyncudp.nim +++ /dev/null @@ -1,78 +0,0 @@ -discard """ - file: "tasyncudp.nim" - output: "2000" -""" -import asyncio, sockets, strutils, times - -const - swarmSize = 5 - messagesToSend = 200 - -var - disp = newDispatcher() - msgCount = 0 - currentClient = 0 - -proc serverRead(s: PAsyncSocket) = - var data = "" - var address = "" - var port: TPort - if s.recvFromAsync(data, 9, address, port): - assert address == "127.0.0.1" - msgCount.inc() - - discard """ - - var line = "" - assert s.recvLine(line) - - if line == "": - assert(false) - else: - if line.startsWith("Message "): - msgCount.inc() - else: - assert(false) - """ - -proc swarmConnect(s: PAsyncSocket) = - for i in 1..messagesToSend: - s.send("Message\c\L") - -proc createClient(disp: var PDispatcher, port: TPort, - buffered = true) = - currentClient.inc() - var client = asyncSocket(typ = SOCK_DGRAM, protocol = IPPROTO_UDP, - buffered = buffered) - client.handleConnect = swarmConnect - disp.register(client) - client.connect("localhost", port) - -proc createServer(port: TPort, buffered = true) = - var server = asyncSocket(typ = SOCK_DGRAM, protocol = IPPROTO_UDP, - buffered = buffered) - server.handleRead = serverRead - disp.register(server) - server.bindAddr(port) - -let serverCount = 2 - -createServer(TPort(10335), false) -createServer(TPort(10336), true) -var startTime = epochTime() -while true: - if epochTime() - startTime >= 300.0: - break - - if not disp.poll(): - break - - if (msgCount div messagesToSend) * serverCount == currentClient: - createClient(disp, TPort(10335), false) - createClient(disp, TPort(10336), true) - - if msgCount == messagesToSend * serverCount * swarmSize: - break - -assert msgCount == messagesToSend * serverCount * swarmSize -echo(msgCount) diff --git a/tests/async/tawaitsemantics.nim b/tests/async/tawaitsemantics.nim index 3e0c3903e..98fb5dfd5 100644 --- a/tests/async/tawaitsemantics.nim +++ b/tests/async/tawaitsemantics.nim @@ -2,17 +2,21 @@ discard """ file: "tawaitsemantics.nim" exitcode: 0 output: ''' -Error caught -Test infix -Test call +Error can be caught using yield +Infix `or` raises +Infix `and` raises +All() raises +Awaiting a async procedure call raises +Awaiting a future raises ''' """ import asyncdispatch # This tests the behaviour of 'await' under different circumstances. -# For example, when awaiting Future variable and this future has failed the -# exception shouldn't be raised as described here +# Specifically, when an awaited future raises an exception then `await` should +# also raise that exception by `read`'ing that future. In cases where you don't +# want this behaviour, you can use `yield`. # https://github.com/nim-lang/Nim/issues/4170 proc thrower(): Future[void] = @@ -23,37 +27,71 @@ proc dummy: Future[void] = result = newFuture[void]() result.complete() -proc testInfix() {.async.} = - # Test the infix operator semantics. +proc testInfixOr() {.async.} = + # Test the infix `or` operator semantics. var fut = thrower() var fut2 = dummy() - await fut or fut2 # Shouldn't raise. - # TODO: what about: await thrower() or fut2? + await fut or fut2 # Should raise! + +proc testInfixAnd() {.async.} = + # Test the infix `and` operator semantics. + var fut = thrower() + var fut2 = dummy() + await fut and fut2 # Should raise! + +proc testAll() {.async.} = + # Test the `all` semantics. + var fut = thrower() + var fut2 = dummy() + await all(fut, fut2) # Should raise! proc testCall() {.async.} = await thrower() +proc testAwaitFut() {.async.} = + var fut = thrower() + await fut # This should raise. + proc tester() {.async.} = # Test that we can handle exceptions without 'try' var fut = thrower() doAssert fut.finished doAssert fut.failed doAssert fut.error.msg == "Test" - await fut # We are awaiting a 'Future', so no `read` occurs. + yield fut # We are yielding a 'Future', so no `read` occurs. doAssert fut.finished doAssert fut.failed doAssert fut.error.msg == "Test" - echo("Error caught") + echo("Error can be caught using yield") + + fut = testInfixOr() + yield fut + doAssert fut.finished + doAssert fut.failed + echo("Infix `or` raises") - fut = testInfix() - await fut + fut = testInfixAnd() + yield fut doAssert fut.finished - doAssert(not fut.failed) - echo("Test infix") + doAssert fut.failed + echo("Infix `and` raises") + + fut = testAll() + yield fut + doAssert fut.finished + doAssert fut.failed + echo("All() raises") fut = testCall() - await fut + yield fut + doAssert fut.failed + echo("Awaiting a async procedure call raises") + + # Test that await will read the future and raise an exception. + fut = testAwaitFut() + yield fut doAssert fut.failed - echo("Test call") + echo("Awaiting a future raises") + waitFor(tester()) diff --git a/tests/async/tfuturevar.nim b/tests/async/tfuturevar.nim new file mode 100644 index 000000000..73c0fddf7 --- /dev/null +++ b/tests/async/tfuturevar.nim @@ -0,0 +1,47 @@ +import asyncdispatch + +proc completeOnReturn(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "" + fut.mget.add("foobar") + return + +proc completeOnImplicitReturn(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "" + fut.mget.add("foobar") + +proc failureTest(fut: FutureVar[string], x: bool) {.async.} = + if x: + raise newException(Exception, "Test") + +proc manualComplete(fut: FutureVar[string], x: bool) {.async.} = + if x: + fut.mget() = "Hello World" + fut.complete() + return + +proc main() {.async.} = + var fut: FutureVar[string] + + fut = newFutureVar[string]() + await completeOnReturn(fut, true) + doAssert(fut.read() == "foobar") + + fut = newFutureVar[string]() + await completeOnImplicitReturn(fut, true) + doAssert(fut.read() == "foobar") + + fut = newFutureVar[string]() + let retFut = failureTest(fut, true) + yield retFut + doAssert(fut.read().isNil) + doAssert(fut.finished) + + fut = newFutureVar[string]() + await manualComplete(fut, true) + doAssert(fut.read() == "Hello World") + + +waitFor main() + diff --git a/tests/async/tioselectors.nim b/tests/async/tioselectors.nim index 2237de01a..71d901e69 100644 --- a/tests/async/tioselectors.nim +++ b/tests/async/tioselectors.nim @@ -217,6 +217,216 @@ when not defined(windows): assert(selector.isEmpty()) result = true + when defined(macosx) or defined(freebsd) or defined(openbsd) or + defined(netbsd): + + proc rename(frompath: cstring, topath: cstring): cint + {.importc: "rename", header: "<stdio.h>".} + + proc createFile(name: string): cint = + result = posix.open(cstring(name), posix.O_CREAT or posix.O_RDWR) + if result == -1: + raiseOsError(osLastError()) + + proc writeFile(name: string, data: string) = + let fd = posix.open(cstring(name), posix.O_APPEND or posix.O_RDWR) + if fd == -1: + raiseOsError(osLastError()) + let length = len(data).cint + if posix.write(fd, cast[pointer](unsafeAddr data[0]), + len(data).cint) != length: + raiseOsError(osLastError()) + if posix.close(fd) == -1: + raiseOsError(osLastError()) + + proc closeFile(fd: cint) = + if posix.close(fd) == -1: + raiseOsError(osLastError()) + + proc removeFile(name: string) = + let err = posix.unlink(cstring(name)) + if err == -1: + raiseOsError(osLastError()) + + proc createDir(name: string) = + let err = posix.mkdir(cstring(name), 0x1FF) + if err == -1: + raiseOsError(osLastError()) + + proc removeDir(name: string) = + let err = posix.rmdir(cstring(name)) + if err == -1: + raiseOsError(osLastError()) + + proc chmodPath(name: string, mode: cint) = + let err = posix.chmod(cstring(name), Mode(mode)) + if err == -1: + raiseOsError(osLastError()) + + proc renameFile(names: string, named: string) = + let err = rename(cstring(names), cstring(named)) + if err == -1: + raiseOsError(osLastError()) + + proc symlink(names: string, named: string) = + let err = posix.symlink(cstring(names), cstring(named)) + if err == -1: + raiseOsError(osLastError()) + + proc openWatch(name: string): cint = + result = posix.open(cstring(name), posix.O_RDONLY) + if result == -1: + raiseOsError(osLastError()) + + const + testDirectory = "/tmp/kqtest" + + type + valType = object + fd: cint + events: set[Event] + + proc vnode_test(): bool = + proc validate[T](test: openarray[ReadyKey[T]], + check: openarray[valType]): bool = + result = false + if len(test) == len(check): + for checkItem in check: + result = false + for testItem in test: + if testItem.fd == checkItem.fd and + checkItem.events <= testItem.events: + result = true + break + if not result: + break + + var res: seq[ReadyKey[int]] + var selector = newSelector[int]() + var events = {Event.VnodeWrite, Event.VnodeDelete, Event.VnodeExtend, + Event.VnodeAttrib, Event.VnodeLink, Event.VnodeRename, + Event.VnodeRevoke} + + result = true + discard posix.unlink(testDirectory) + + createDir(testDirectory) + var dirfd = posix.open(cstring(testDirectory), posix.O_RDONLY) + if dirfd == -1: + raiseOsError(osLastError()) + + selector.registerVnode(dirfd, events, 1) + selector.flush() + + # chmod testDirectory to 0777 + chmodPath(testDirectory, 0x1FF) + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeAttrib} <= res[0].events) + + # create subdirectory + createDir(testDirectory & "/test") + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeWrite, + Event.VnodeLink} <= res[0].events) + + # open test directory for watching + var testfd = openWatch(testDirectory & "/test") + selector.registerVnode(testfd, events, 2) + selector.flush() + doAssert(len(selector.select(0)) == 0) + + # rename test directory + renameFile(testDirectory & "/test", testDirectory & "/renamed") + res = selector.select(0) + doAssert(len(res) == 2) + doAssert(len(selector.select(0)) == 0) + doAssert(validate(res, + [valType(fd: dirfd, events: {Event.Vnode, Event.VnodeWrite}), + valType(fd: testfd, + events: {Event.Vnode, Event.VnodeRename})]) + ) + + # remove test directory + removeDir(testDirectory & "/renamed") + res = selector.select(0) + doAssert(len(res) == 2) + doAssert(len(selector.select(0)) == 0) + doAssert(validate(res, + [valType(fd: dirfd, events: {Event.Vnode, Event.VnodeWrite, + Event.VnodeLink}), + valType(fd: testfd, + events: {Event.Vnode, Event.VnodeDelete})]) + ) + # create file new test file + testfd = createFile(testDirectory & "/testfile") + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeWrite} <= res[0].events) + + # close new test file + closeFile(testfd) + doAssert(len(selector.select(0)) == 0) + doAssert(len(selector.select(0)) == 0) + + # chmod test file with 0666 + chmodPath(testDirectory & "/testfile", 0x1B6) + doAssert(len(selector.select(0)) == 0) + + testfd = openWatch(testDirectory & "/testfile") + selector.registerVnode(testfd, events, 1) + selector.flush() + + # write data to test file + writeFile(testDirectory & "/testfile", "TESTDATA") + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == testfd and + {Event.Vnode, Event.VnodeWrite, + Event.VnodeExtend} <= res[0].events) + + # symlink test file + symlink(testDirectory & "/testfile", testDirectory & "/testlink") + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeWrite} <= res[0].events) + + # remove test file + removeFile(testDirectory & "/testfile") + res = selector.select(0) + doAssert(len(res) == 2) + doAssert(len(selector.select(0)) == 0) + doAssert(validate(res, + [valType(fd: testfd, events: {Event.Vnode, Event.VnodeDelete}), + valType(fd: dirfd, events: {Event.Vnode, Event.VnodeWrite})]) + ) + + # remove symlink + removeFile(testDirectory & "/testlink") + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeWrite} <= res[0].events) + + # remove testDirectory + removeDir(testDirectory) + res = selector.select(0) + doAssert(len(res) == 1) + doAssert(len(selector.select(0)) == 0) + doAssert(res[0].fd == dirfd and + {Event.Vnode, Event.VnodeDelete} <= res[0].events) + when hasThreadSupport: var counter = 0 @@ -256,6 +466,9 @@ when not defined(windows): processTest("Timer notification test...", timer_notification_test()) processTest("Process notification test...", process_notification_test()) processTest("Signal notification test...", signal_notification_test()) + when defined(macosx) or defined(freebsd) or defined(openbsd) or + defined(netbsd): + processTest("File notification test...", vnode_test()) echo("All tests passed!") else: import nativesockets, winlean, os, osproc diff --git a/tests/async/tmultisync.nim b/tests/async/tmultisync.nim new file mode 100644 index 000000000..9ef9b105c --- /dev/null +++ b/tests/async/tmultisync.nim @@ -0,0 +1,8 @@ +import asyncdispatch, net, asyncnet + +proc recvTwice(socket: Socket | AsyncSocket, + size: int): Future[string] {.multisync.} = + var x = await socket.recv(size) + var y = await socket.recv(size+1) + return x & "aboo" & y + diff --git a/tests/async/tnewasyncudp.nim b/tests/async/tnewasyncudp.nim index 7025fa20d..6277e877c 100644 --- a/tests/async/tnewasyncudp.nim +++ b/tests/async/tnewasyncudp.nim @@ -39,8 +39,11 @@ proc prepareAddress(intaddr: uint32, intport: uint16): ptr Sockaddr_in = proc launchSwarm(name: ptr SockAddr) {.async.} = var i = 0 var k = 0 + var buffer: array[16384, char] + var slen = sizeof(Sockaddr_in).SockLen + var saddr = Sockaddr_in() while i < swarmSize: - var peeraddr = prepareAddress(INADDR_ANY, 0) + var peeraddr = prepareAddress(INADDR_LOOPBACK, 0) var sock = newAsyncNativeSocket(nativesockets.AF_INET, nativesockets.SOCK_DGRAM, Protocol.IPPROTO_UDP) @@ -50,10 +53,19 @@ proc launchSwarm(name: ptr SockAddr) {.async.} = let sockport = getSockName(sock.SocketHandle).int k = 0 while k < messagesToSend: + zeroMem(addr(buffer[0]), 16384) + zeroMem(cast[pointer](addr(saddr)), sizeof(Sockaddr_in)) var message = "Message " & $(i * messagesToSend + k) await sendTo(sock, addr message[0], len(message), name, sizeof(Sockaddr_in).SockLen) - saveSendingPort(sockport) + var size = await recvFromInto(sock, cast[pointer](addr buffer[0]), + 16384, cast[ptr SockAddr](addr saddr), + addr slen) + size = 0 + var grammString = $buffer + if grammString == message: + saveSendingPort(sockport) + inc(recvCount) inc(k) closeSocket(sock) inc(i) @@ -75,13 +87,14 @@ proc readMessages(server: AsyncFD) {.async.} = var grammString = $buffer if grammString.startswith("Message ") and saddr.sin_addr.s_addr == 0x100007F: + await sendTo(server, addr grammString[0], len(grammString), + cast[ptr SockAddr](addr saddr), slen) inc(msgCount) saveReceivedPort(ntohs(saddr.sin_port).int) - inc(recvCount) inc(i) proc createServer() {.async.} = - var name = prepareAddress(INADDR_ANY, serverPort) + var name = prepareAddress(INADDR_LOOPBACK, serverPort) var server = newAsyncNativeSocket(nativesockets.AF_INET, nativesockets.SOCK_DGRAM, Protocol.IPPROTO_UDP) @@ -90,7 +103,7 @@ proc createServer() {.async.} = raiseOSError(osLastError()) asyncCheck readMessages(server) -var name = prepareAddress(0x7F000001, serverPort) # 127.0.0.1 +var name = prepareAddress(INADDR_LOOPBACK, serverPort) # 127.0.0.1 asyncCheck createServer() asyncCheck launchSwarm(cast[ptr SockAddr](name)) while true: diff --git a/tests/async/tpolltimeouts.nim b/tests/async/tpolltimeouts.nim new file mode 100644 index 000000000..dac33732d --- /dev/null +++ b/tests/async/tpolltimeouts.nim @@ -0,0 +1,19 @@ +discard """ + output: "true" +""" +# Issue https://github.com/nim-lang/Nim/issues/4262 +import asyncdispatch, times + +proc foo(): Future[int] {.async.} = + return 1 + +proc bar(): Future[int] {.async.} = + return await foo() + +let start = epochTime() +let barFut = bar() + +while not barFut.finished: + poll(2000) + +echo(epochTime() - start < 1.0) diff --git a/tests/async/tupcoming_async.nim b/tests/async/tupcoming_async.nim new file mode 100644 index 000000000..137794afd --- /dev/null +++ b/tests/async/tupcoming_async.nim @@ -0,0 +1,114 @@ +discard """ + output: ''' +OK +OK +OK +OK +''' +""" + +when defined(upcoming): + import asyncdispatch, times, osproc, streams + + const supportedPlatform = defined(linux) or defined(freebsd) or + defined(netbsd) or defined(openbsd) or + defined(macosx) + + proc waitEvent(ev: AsyncEvent, closeEvent = false): Future[void] = + var retFuture = newFuture[void]("waitEvent") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + if closeEvent: + return true + else: + return false + addEvent(ev, cb) + return retFuture + + proc waitTimer(timeout: int): Future[void] = + var retFuture = newFuture[void]("waitTimer") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + addTimer(timeout, true, cb) + return retFuture + + proc waitProcess(p: Process): Future[void] = + var retFuture = newFuture[void]("waitProcess") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + addProcess(p.processID(), cb) + return retFuture + + proc delayedSet(ev: AsyncEvent, timeout: int): Future[void] {.async.} = + await waitTimer(timeout) + ev.setEvent() + + proc timerTest() = + waitFor(waitTimer(200)) + echo "OK" + + proc eventTest() = + var event = newAsyncEvent() + var fut = waitEvent(event) + asyncCheck(delayedSet(event, 500)) + waitFor(fut or waitTimer(1000)) + if fut.finished: + echo "OK" + else: + echo "eventTest: Timeout expired before event received!" + + proc processTest() = + when defined(windows): + var process = startProcess("ping.exe", "", + ["127.0.0.1", "-n", "2", "-w", "100"], nil, + {poStdErrToStdOut, poUsePath, poInteractive, + poDemon}) + else: + var process = startProcess("/bin/sleep", "", ["1"], nil, + {poStdErrToStdOut, poUsePath}) + var fut = waitProcess(process) + waitFor(fut or waitTimer(2000)) + if fut.finished and process.peekExitCode() == 0: + echo "OK" + else: + echo "processTest: Timeout expired before process exited!" + + when supportedPlatform: + import posix + + proc waitSignal(signal: int): Future[void] = + var retFuture = newFuture[void]("waitSignal") + proc cb(fd: AsyncFD): bool = + retFuture.complete() + addSignal(signal, cb) + return retFuture + + proc delayedSignal(signal: int, timeout: int): Future[void] {.async.} = + await waitTimer(timeout) + var pid = posix.getpid() + discard posix.kill(pid, signal.cint) + + proc signalTest() = + var fut = waitSignal(posix.SIGINT) + asyncCheck(delayedSignal(posix.SIGINT, 500)) + waitFor(fut or waitTimer(1000)) + if fut.finished: + echo "OK" + else: + echo "signalTest: Timeout expired before signal received!" + + when supportedPlatform: + timerTest() + eventTest() + processTest() + signalTest() + elif defined(windows): + timerTest() + eventTest() + processTest() + echo "OK" + else: + eventTest() + echo "OK\nOK\nOK" +else: + echo "OK\nOK\nOK\nOK" diff --git a/tests/bind/tnicerrorforsymchoice.nim b/tests/bind/tnicerrorforsymchoice.nim index 5145fdcff..bd00188fa 100644 --- a/tests/bind/tnicerrorforsymchoice.nim +++ b/tests/bind/tnicerrorforsymchoice.nim @@ -1,6 +1,6 @@ discard """ line: 18 - errormsg: "type mismatch: got (proc (s: TScgi) | proc (client: AsyncSocket, headers: StringTableRef, input: string){.gcsafe, locks: 0.}" + errormsg: "type mismatch: got (proc (s: TScgi) | proc (client: AsyncSocket, headers: StringTableRef, input: string){.noSideEffect, gcsafe, locks: 0.}" """ #bug #442 diff --git a/tests/ccgbugs/tescaping_temps.nim b/tests/ccgbugs/tescaping_temps.nim index ef078913b..ea09261ea 100644 --- a/tests/ccgbugs/tescaping_temps.nim +++ b/tests/ccgbugs/tescaping_temps.nim @@ -18,3 +18,14 @@ test proc() = test proc() = discard else: test proc() = discard + +# ensure 'case' does not trigger the same bug: +test proc() = + let f = 15 + case f + of 10: + test proc() = discard + of 3: + test proc() = discard + else: + test proc() = discard diff --git a/tests/ccgbugs/tuple_canon.nim b/tests/ccgbugs/tuple_canon.nim index 960e2aae9..a607f9cab 100644 --- a/tests/ccgbugs/tuple_canon.nim +++ b/tests/ccgbugs/tuple_canon.nim @@ -1,3 +1,10 @@ + + +# bug #4626 +var foo: (int, array[1, int]) # Tuple must be of length > 1 +let bar = (1, [1]) +foo = bar # No error if assigned directly + # bug #2250 import diff --git a/tests/closure/tclosure4.nim b/tests/closure/tclosure4.nim index 69c076cd5..bc134ded6 100644 --- a/tests/closure/tclosure4.nim +++ b/tests/closure/tclosure4.nim @@ -1,7 +1,7 @@ import json, tables, sequtils -proc run(json_params: Table) = +proc run(json_params: OrderedTable) = let json_elems = json_params["files"].elems # These fail compilation. var files = map(json_elems, proc (x: JsonNode): string = x.str) diff --git a/tests/collections/tableadds.nim b/tests/collections/tableadds.nim new file mode 100644 index 000000000..71f1fad7d --- /dev/null +++ b/tests/collections/tableadds.nim @@ -0,0 +1,13 @@ +discard """ + output: '''done''' +""" + +import tables + +proc main = + var tab = newTable[string, string]() + for i in 0..1000: + tab.add "key", "value " & $i + +main() +echo "done" diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index a8a182a78..59fef4920 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -134,6 +134,29 @@ block mpairsTableTest1: block SyntaxTest: var x = toTable[int, string]({:}) +# Until #4448 is fixed, these tests will fail +when false: + block clearTableTest: + var t = data.toTable + assert t.len() != 0 + t.clear() + assert t.len() == 0 + + block clearOrderedTableTest: + var t = data.toOrderedTable + assert t.len() != 0 + t.clear() + assert t.len() == 0 + + block clearCountTableTest: + var t = initCountTable[string]() + t.inc("90", 3) + t.inc("12", 2) + t.inc("34", 1) + assert t.len() != 0 + t.clear() + assert t.len() == 0 + proc orderedTableSortTest() = var t = initOrderedTable[string, int](2) for key, val in items(data): t[key] = val diff --git a/tests/collections/ttablesref.nim b/tests/collections/ttablesref.nim index 32494f1f2..12af1ccbb 100644 --- a/tests/collections/ttablesref.nim +++ b/tests/collections/ttablesref.nim @@ -141,6 +141,31 @@ block anonZipTest: let values = @[1, 2, 3] doAssert "{a: 1, b: 2, c: 3}" == $ toTable zip(keys, values) +block clearTableTest: + var t = newTable[string, float]() + t["test"] = 1.2345 + t["111"] = 1.000043 + t["123"] = 1.23 + assert t.len() != 0 + t.clear() + assert t.len() == 0 + +block clearOrderedTableTest: + var t = newOrderedTable[string, int](2) + for key, val in items(data): t[key] = val + assert t.len() != 0 + t.clear() + assert t.len() == 0 + +block clearCountTableTest: + var t = newCountTable[string]() + t.inc("90", 3) + t.inc("12", 2) + t.inc("34", 1) + assert t.len() != 0 + t.clear() + assert t.len() == 0 + orderedTableSortTest() echo "true" diff --git a/tests/cpp/tempty_generic_obj.nim b/tests/cpp/tempty_generic_obj.nim new file mode 100644 index 000000000..e2957a5cd --- /dev/null +++ b/tests/cpp/tempty_generic_obj.nim @@ -0,0 +1,22 @@ +discard """ + cmd: "nim cpp $file" + output: '''int +float''' +""" + +import typetraits + +# bug #4625 +type + Vector {.importcpp: "std::vector<'0 >", header: "vector".} [T] = object + +proc initVector[T](): Vector[T] {.importcpp: "'0(@)", header: "vector", constructor.} + +proc doSomething[T](v: var Vector[T]) = + echo T.name + +var v = initVector[int]() +v.doSomething() + +var vf = initVector[float]() +vf.doSomething() # Nim uses doSomething[int] here in C++ diff --git a/tests/distinct/tcurrncy.nim b/tests/distinct/tcurrncy.nim index 7ad4caea4..2675de739 100644 --- a/tests/distinct/tcurrncy.nim +++ b/tests/distinct/tcurrncy.nim @@ -2,7 +2,7 @@ discard """ file: "tcurrncy.nim" output: "25" """ -template Additive(typ: typeDesc): stmt = +template Additive(typ: untyped) = proc `+` *(x, y: typ): typ {.borrow.} proc `-` *(x, y: typ): typ {.borrow.} @@ -10,18 +10,18 @@ template Additive(typ: typeDesc): stmt = proc `+` *(x: typ): typ {.borrow.} proc `-` *(x: typ): typ {.borrow.} -template Multiplicative(typ, base: typeDesc): stmt {.immediate.} = +template Multiplicative(typ, base: untyped) = proc `*` *(x: typ, y: base): typ {.borrow.} proc `*` *(x: base, y: typ): typ {.borrow.} proc `div` *(x: typ, y: base): typ {.borrow.} proc `mod` *(x: typ, y: base): typ {.borrow.} -template Comparable(typ: typeDesc): stmt = +template Comparable(typ: untyped) = proc `<` * (x, y: typ): bool {.borrow.} proc `<=` * (x, y: typ): bool {.borrow.} proc `==` * (x, y: typ): bool {.borrow.} -template DefineCurrency(typ, base: expr): stmt {.immediate.} = +template DefineCurrency(typ, base: untyped) = type typ* = distinct base Additive(typ) diff --git a/tests/effects/tsidee4.nim b/tests/effects/tsidee4.nim index 2cb88a23e..ecc79580c 100644 --- a/tests/effects/tsidee4.nim +++ b/tests/effects/tsidee4.nim @@ -1,13 +1,13 @@ discard """ file: "tsidee4.nim" - line: 15 - errormsg: "type mismatch" + line: 12 + errormsg: "'noSideEffect' can have side effects" """ var global: int -proc dontcare(x: int): int = return x +proc dontcare(x: int): int = return global proc noSideEffect(x, y: int, p: proc (a: int): int {.noSideEffect.}): int {.noSideEffect.} = return x + y + dontcare(x) diff --git a/tests/exception/texceptions.nim b/tests/exception/texceptions.nim index bdf338599..b30b3874b 100644 --- a/tests/exception/texceptions.nim +++ b/tests/exception/texceptions.nim @@ -9,7 +9,7 @@ FINALLY RECOVER BEFORE -EXCEPT +EXCEPT: IOError: hi FINALLY ''' """ @@ -52,10 +52,10 @@ echo "" proc return_in_except = try: echo "BEFORE" - raise newException(IOError, "") + raise newException(IOError, "hi") except: - echo "EXCEPT" + echo "EXCEPT: ", getCurrentException().name, ": ", getCurrentExceptionMsg() return finally: diff --git a/tests/exception/tunhandledexc.nim b/tests/exception/tunhandledexc.nim index 63a402414..c318aec81 100644 --- a/tests/exception/tunhandledexc.nim +++ b/tests/exception/tunhandledexc.nim @@ -14,10 +14,9 @@ proc genErrors(s: string) = raise newException(EsomeotherErr, "bla") when true: + try: discard except: discard + try: genErrors("errssor!") except ESomething: echo("Error happened") - - - diff --git a/tests/float/tfloat4.nim b/tests/float/tfloat4.nim index 006b4d88f..d7783ce26 100644 --- a/tests/float/tfloat4.nim +++ b/tests/float/tfloat4.nim @@ -30,8 +30,10 @@ let testFloats = [ "0.00097656250000000021684043449710088680149056017398834228515625" ] -for num in testFloats: - doAssert num.parseFloat.floatToStr.parseFloat == num.parseFloat +when not defined(windows): + # Windows' sprintf produces niceties like -1.#INF... + for num in testFloats: + doAssert num.parseFloat.floatToStr.parseFloat == num.parseFloat doAssert "0".parseFloat == 0.0 doAssert "-.1".parseFloat == -0.1 diff --git a/tests/generics/tlamba_in_generic.nim b/tests/generics/tlamba_in_generic.nim new file mode 100644 index 000000000..91d417b5e --- /dev/null +++ b/tests/generics/tlamba_in_generic.nim @@ -0,0 +1,13 @@ +discard """ + output: '''!!Hi!!''' +""" +# bug #4658 +import future + +var x = 123 + +proc twice[T](f: T -> T): T -> T = (x: T) => f(f(x)) + +proc quote(s: string): string = "!" & s & "!" + +echo twice(quote)("Hi") diff --git a/tests/generics/ttypeclass_to_typeclass.nim b/tests/generics/ttypeclass_to_typeclass.nim new file mode 100644 index 000000000..a5f010450 --- /dev/null +++ b/tests/generics/ttypeclass_to_typeclass.nim @@ -0,0 +1,12 @@ +# bug #4672 +type + EnumContainer[T: enum] = object + v: T + SomeEnum {.pure.} = enum + A,B,C + +proc value[T: enum](this: EnumContainer[T]): T = + this.v + +var enumContainer: EnumContainer[SomeEnum] +discard enumContainer.value() diff --git a/tests/import_in_config/nim.cfg b/tests/import_in_config/nim.cfg new file mode 100644 index 000000000..af112d112 --- /dev/null +++ b/tests/import_in_config/nim.cfg @@ -0,0 +1,2 @@ +import = "other" +path = "$projectDir" diff --git a/tests/import_in_config/other.nim b/tests/import_in_config/other.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/import_in_config/other.nim diff --git a/tests/import_in_config/tmain.nim b/tests/import_in_config/tmain.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/import_in_config/tmain.nim diff --git a/tests/js/taddr.nim b/tests/js/taddr.nim index 1fba30d55..d8ff4c11b 100644 --- a/tests/js/taddr.nim +++ b/tests/js/taddr.nim @@ -70,3 +70,10 @@ proc testPtr(p: pointer, a: int) = var i = 123 testPtr(addr i, 5) doAssert(i == 124) + +var someGlobal = 5 +proc getSomeGlobalPtr(): ptr int = addr someGlobal +let someGlobalPtr = getSomeGlobalPtr() +doAssert(someGlobalPtr[] == 5) +someGlobalPtr[] = 10 +doAssert(someGlobal == 10) diff --git a/tests/js/tbyvar.nim b/tests/js/tbyvar.nim index 9714cd56b..40aebd13b 100644 --- a/tests/js/tbyvar.nim +++ b/tests/js/tbyvar.nim @@ -41,3 +41,11 @@ proc bar(s: var seq[int], a: int) = foo(s) s.bar(5) doAssert(s == @[123, 1]) + +import tables +block: # Test get addr of byvar return value + var t = initTable[string, int]() + t["hi"] = 5 + let a = addr t["hi"] + a[] = 10 + doAssert(t["hi"] == 10) diff --git a/tests/js/tunittests.nim b/tests/js/tunittests.nim index 4b09c99a9..7c2e70563 100644 --- a/tests/js/tunittests.nim +++ b/tests/js/tunittests.nim @@ -1,5 +1,7 @@ discard """ - output: '''[OK] >:)''' + output: ''' +[Suite] Bacon + [OK] >:)''' """ import unittest diff --git a/tests/macros/tgettype2.nim b/tests/macros/tgettype2.nim new file mode 100644 index 000000000..f129e6e1b --- /dev/null +++ b/tests/macros/tgettype2.nim @@ -0,0 +1,67 @@ + +import macros, typetraits + +type Foo = distinct int +type Bar = distinct int +type Baz = int + +let foo = 0.Foo +let bar = 1.Bar +let baz = 2.Baz + +type MyType[T] = distinct tuple[a,b:T] +type MySimpleType = distinct tuple[a,b: int] + +var v: seq[int] +var vv: seq[float] +var t: MyType[int] +var tt: MyType[float] +var s: MySimpleType + +echo "############" +echo "#### gt ####" +echo "############" + +macro gt(a: typed): string = + let b = a.getType + var str = "gt(" & $a & "):\t" & b.repr + if b.kind == nnkSym: # bad predicat to check weather the type has an implementation + str = str & ", " & b.getType.repr # append the implementation to the result + result = newLit(str) + +echo gt(Foo) # typeDesc[Foo] +echo gt(Bar) # typeDesc[Bar] +echo gt(Baz) # typeDesc[int] shouldn't it be typeDesc[Baz]? +echo gt(foo) # distinct[int] I would prefer Foo, distinct[int] +echo gt(bar) # distinct[int] I would prefer Bar, distinct[int] +echo gt(baz) # int, int I would prefer Baz, int + +echo gt(v) # seq[int], ok +echo gt(vv) # seq[float], ok +echo gt(t) # MyType, distinct[tuple[int, int]] I would prefer MyType[int], distinct[tuple[int, int]] +echo gt(tt) # MyType, distinct[tuple[float, float]] I would prefer MyType[float], distinct[tuple[int, int]] +echo gt(s) # distinct[tuple[int, int]] I would prefer MySimpleType, distinct[tuple[int,int]] + +echo "#############" +echo "#### gt2 ####" +echo "#############" + +# get type name via typetraits + +macro gt2(a: typed): string = + let prefix = "gt2(" & $a & "): \t" + result = quote do: + `prefix` & `a`.type.name + +echo gt2(Foo) # Foo shouldn't this be typeDesc[Foo] ? +echo gt2(Bar) # Bar shouldn't this be typeDesc[Bar] ? +echo gt2(Baz) # Baz shouldn't this be typeDesc[Baz] ? +echo gt2(foo) # Foo +echo gt2(bar) # Bar +echo gt2(baz) # Baz + +echo gt2(v) # seq[int] +echo gt2(vv) # seq[float] +echo gt2(t) # MyType[system.int] why is it system.int and not just int like in seq? +echo gt2(tt) # MyType[system.float] why is it system.float and not just float like in seq? +echo gt2(s) # MySimpleType diff --git a/tests/manyloc/keineschweine/README.md b/tests/manyloc/keineschweine/README.md index 1323f4ae3..20facbced 100644 --- a/tests/manyloc/keineschweine/README.md +++ b/tests/manyloc/keineschweine/README.md @@ -11,7 +11,7 @@ Just a dumb little game ### How to build? -* `git clone --recursive git://github.com/fowlmouth/keineSchweine.git somedir` +* `git clone --recursive https://github.com/fowlmouth/keineSchweine.git somedir` * `cd somedir` * `nim c -r nakefile test` or `nim c -r keineschweine && ./keineschweine` diff --git a/tests/metatype/twildtypedesc.nim b/tests/metatype/twildtypedesc.nim new file mode 100644 index 000000000..268bff0d8 --- /dev/null +++ b/tests/metatype/twildtypedesc.nim @@ -0,0 +1,43 @@ +discard """ + output: '''123 +123 +123 +123 +123 +123''' +""" + +import strutils + +proc unpack(t: typedesc[string], v: string): string = $v +proc unpack(t: typedesc[int], v: string): int = parseInt(v) + +proc unpack[T](v: string): T = + unpack T, v + +var s = "123" + +assert(unpack[string](s) is string) +assert(unpack[int](s) is int) + +echo unpack[int](s) +echo unpack[string](s) + +echo unpack(int,s) +echo unpack(string,s) + +template `as`*(x: untyped, t: typedesc): untyped = unpack(t,x) + +echo s as int +echo s as string + +# bug #4534 + +proc unit(t: typedesc[int]): t = 0 +proc unit(t: typedesc[string]): t = "" +proc unit(t: typedesc[float]): t = 0.0 + +assert unit(int) == 0 +assert unit(string) == "" +assert unit(float) == 0.0 + diff --git a/tests/objects/tinherit_from_generic.nim b/tests/objects/tinherit_from_generic.nim new file mode 100644 index 000000000..6e0e929ce --- /dev/null +++ b/tests/objects/tinherit_from_generic.nim @@ -0,0 +1,13 @@ +discard """ + output: '''true''' +""" + +# bug #4673 +type + BaseObj[T] = ref object of RootObj + SomeObj = ref object of BaseObj[int] + +proc doSomething[T](o: BaseObj[T]) = + echo "true" +var o = new(SomeObj) +o.doSomething() # Error: cannot instantiate: 'T' diff --git a/tests/osproc/tafalse.nim b/tests/osproc/tafalse.nim new file mode 100644 index 000000000..24fd4fb2e --- /dev/null +++ b/tests/osproc/tafalse.nim @@ -0,0 +1,3 @@ +# 'tafalse.nim' to ensure it is compiled before texitcode.nim +import system +quit(QuitFailure) diff --git a/tests/osproc/texitcode.nim b/tests/osproc/texitcode.nim new file mode 100644 index 000000000..1e83658c2 --- /dev/null +++ b/tests/osproc/texitcode.nim @@ -0,0 +1,18 @@ +discard """ + file: "texitcode.nim" + output: "" +""" +import osproc, os + +const filename = when defined(Windows): "tafalse.exe" else: "tafalse" +let dir = getCurrentDir() / "tests" / "osproc" +doAssert fileExists(dir / filename) + +var p = startProcess(filename, dir) +doAssert(waitForExit(p) == QuitFailure) + +p = startProcess(filename, dir) +var running = true +while running: + running = running(p) +doAssert(waitForExit(p) == QuitFailure) diff --git a/tests/overload/importA.nim b/tests/overload/importA.nim new file mode 100644 index 000000000..f045d11b4 --- /dev/null +++ b/tests/overload/importA.nim @@ -0,0 +1,5 @@ +type + Field* = object + elemSize*: int + +template `+`*(x: untyped, y: Field): untyped = x diff --git a/tests/overload/importB.nim b/tests/overload/importB.nim new file mode 100644 index 000000000..2dc3adf7a --- /dev/null +++ b/tests/overload/importB.nim @@ -0,0 +1,15 @@ +type + Foo*[T] = object + v*: T + +template `+`*(x: Foo, y: Foo): untyped = x + +template newvar*(r: untyped): untyped {.dirty.} = + var r: float + +template t1*(x: Foo): untyped = + newvar(y1) + x +template t2*(x: Foo): untyped = + newvar(y2) + x diff --git a/tests/overload/timport.nim b/tests/overload/timport.nim new file mode 100644 index 000000000..8ea65e54d --- /dev/null +++ b/tests/overload/timport.nim @@ -0,0 +1,7 @@ +# issue 4675 +import importA # comment this out to make it work +import importB + +var x: Foo[float] +var y: Foo[float] +let r = t1(x) + t2(y) diff --git a/tests/overload/tselfderef.nim b/tests/overload/tselfderef.nim new file mode 100644 index 000000000..708e4043b --- /dev/null +++ b/tests/overload/tselfderef.nim @@ -0,0 +1,17 @@ +# bug #4671 +{.experimental.} +{.this: self.} + +type + SomeObj = object + f: int + +proc f(num: int) = + discard + +var intptr: ptr int +intptr.f() # compiles fine + +proc doSomething(self: var SomeObj) = + var pint: ptr int + pint.f() # Error: expression '.(pint, "f")' cannot be called diff --git a/tests/parallel/tsendtwice.nim b/tests/parallel/tsendtwice.nim new file mode 100644 index 000000000..0700fc4da --- /dev/null +++ b/tests/parallel/tsendtwice.nim @@ -0,0 +1,71 @@ +discard """ + output: '''obj2 nil +obj nil +obj3 nil +3 +obj2 nil +obj nil +obj3 nil''' + cmd: "nim c -r --threads:on $file" +""" + +# bug #4776 + +import tables + +type + Base* = ref object of RootObj + someSeq: seq[int] + baseData: array[400000, byte] + Derived* = ref object of Base + data: array[400000, byte] + +type + ThreadPool = ref object + threads: seq[ptr Thread[ThreadArg]] + channels: seq[ThreadArg] + TableChannel = Channel[TableRef[string, Base]] + ThreadArg = ptr TableChannel + +var globalTable {.threadvar.}: TableRef[string, Base] +globalTable = newTable[string, Base]() +let d = new(Derived) +globalTable.add("obj", d) +globalTable.add("obj2", d) +globalTable.add("obj3", d) + +proc testThread(channel: ptr TableChannel) {.thread.} = + globalTable = channel[].recv() + for k, v in pairs globaltable: + echo k, " ", v.someSeq + var myObj: Base + deepCopy(myObj, globalTable["obj"]) + myObj.someSeq = newSeq[int](100) + let table = channel[].recv() # same table + echo table.len + for k, v in mpairs table: + echo k, " ", v.someSeq + assert(table.contains("obj")) # fails! + assert(table.contains("obj2")) # fails! + assert(table.contains("obj3")) # fails! + +var channel: TableChannel + +proc newThreadPool(threadCount: int) = #: ThreadPool = + #new(result) + #result.threads = newSeq[ptr Thread[ThreadArg]](threadCount) + #var channel = cast[ptr TableChannel](allocShared0(sizeof(TableChannel))) + channel.open() + channel.send(globalTable) + channel.send(globalTable) + #createThread(threadPtr[], testThread, addr channel) + testThread(addr channel) + #result.threads[i] = threadPtr + +proc stop(p: ThreadPool) = + for t in p.threads: + joinThread(t[]) + dealloc(t) + + +newThreadPool(1)#.stop() diff --git a/tests/rodfiles/amethods.nim b/tests/rodfiles/amethods.nim index ecd36d491..29cf757f7 100644 --- a/tests/rodfiles/amethods.nim +++ b/tests/rodfiles/amethods.nim @@ -1,11 +1,11 @@ type - TBaseClass* = object of TObject + TBaseClass* = object of RootObj proc newBaseClass*: ref TBaseClass = new result -method echoType*(x: ref TBaseClass) = +method echoType*(x: ref TBaseClass) {.base.} = echo "base class" proc echoAlias*(x: ref TBaseClass) = diff --git a/tests/rodfiles/gtkex1.nim b/tests/rodfiles/gtkex1.nim index 156ba5322..50779cb9e 100644 --- a/tests/rodfiles/gtkex1.nim +++ b/tests/rodfiles/gtkex1.nim @@ -1,12 +1,12 @@ import cairo, glib2, gtk2 -proc destroy(widget: pWidget, data: pgpointer) {.cdecl.} = +proc destroy(widget: PWidget, data: Pgpointer) {.cdecl.} = main_quit() var - window: pWidget -nimrod_init() + window: PWidget +nim_init() window = window_new(WINDOW_TOPLEVEL) discard signal_connect(window, "destroy", SIGNAL_FUNC(gtkex1.destroy), nil) diff --git a/tests/rodfiles/gtkex2.nim b/tests/rodfiles/gtkex2.nim index 70926bd50..0949e4872 100644 --- a/tests/rodfiles/gtkex2.nim +++ b/tests/rodfiles/gtkex2.nim @@ -2,17 +2,17 @@ import glib2, gtk2 -proc destroy(widget: pWidget, data: pgpointer){.cdecl.} = +proc destroy(widget: PWidget, data: Pgpointer){.cdecl.} = main_quit() var window: PWidget button: PWidget -nimrod_init() +nim_init() window = window_new(WINDOW_TOPLEVEL) button = button_new("Click me") -set_border_width(PContainer(Window), 5) +set_border_width(PContainer(window), 5) add(PContainer(window), button) discard signal_connect(window, "destroy", SIGNAL_FUNC(gtkex2.destroy), nil) diff --git a/tests/rodfiles/int2bool.nim b/tests/rodfiles/int2bool.nim index 0f6fd14e6..bb0682844 100644 --- a/tests/rodfiles/int2bool.nim +++ b/tests/rodfiles/int2bool.nim @@ -2,7 +2,6 @@ {.overflowchecks: on.} converter uglyToBool*(x: int): bool = - {.Breakpoint.} result = x != 0 diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim new file mode 100644 index 000000000..dd9a6139a --- /dev/null +++ b/tests/stdlib/thttpclient.nim @@ -0,0 +1,113 @@ +discard """ + cmd: "nim c -d:ssl $file" +""" + +import strutils +from net import TimeoutError + +import httpclient, asyncdispatch + +const manualTests = false + +proc asyncTest() {.async.} = + var client = newAsyncHttpClient() + var resp = await client.request("http://example.com/") + doAssert(resp.code.is2xx) + doAssert("<title>Example Domain</title>" in resp.body) + + resp = await client.request("http://example.com/404") + doAssert(resp.code.is4xx) + doAssert(resp.code == Http404) + doAssert(resp.status == Http404) + + resp = await client.request("https://google.com/") + doAssert(resp.code.is2xx or resp.code.is3xx) + + # getContent + try: + discard await client.getContent("https://google.com/404") + doAssert(false, "HttpRequestError should have been raised") + except HttpRequestError: + discard + except: + doAssert(false, "HttpRequestError should have been raised") + + + # Multipart test. + var data = newMultipartData() + data["output"] = "soap12" + data["uploaded_file"] = ("test.html", "text/html", + "<html><head></head><body><p>test</p></body></html>") + resp = await client.post("http://validator.w3.org/check", multipart=data) + doAssert(resp.code.is2xx) + + # onProgressChanged + when manualTests: + proc onProgressChanged(total, progress, speed: BiggestInt) {.async.} = + echo("Downloaded ", progress, " of ", total) + echo("Current rate: ", speed div 1000, "kb/s") + client.onProgressChanged = onProgressChanged + discard await client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") + + client.close() + + # Proxy test + #when manualTests: + # client = newAsyncHttpClient(proxy = newProxy("http://51.254.106.76:80/")) + # var resp = await client.request("https://github.com") + # echo resp + +proc syncTest() = + var client = newHttpClient() + var resp = client.request("http://example.com/") + doAssert(resp.code.is2xx) + doAssert("<title>Example Domain</title>" in resp.body) + + resp = client.request("http://example.com/404") + doAssert(resp.code.is4xx) + doAssert(resp.code == Http404) + doAssert(resp.status == Http404) + + resp = client.request("https://google.com/") + doAssert(resp.code.is2xx or resp.code.is3xx) + + # getContent + try: + discard client.getContent("https://google.com/404") + doAssert(false, "HttpRequestError should have been raised") + except HttpRequestError: + discard + except: + doAssert(false, "HttpRequestError should have been raised") + + # Multipart test. + var data = newMultipartData() + data["output"] = "soap12" + data["uploaded_file"] = ("test.html", "text/html", + "<html><head></head><body><p>test</p></body></html>") + resp = client.post("http://validator.w3.org/check", multipart=data) + doAssert(resp.code.is2xx) + + # onProgressChanged + when manualTests: + proc onProgressChanged(total, progress, speed: BiggestInt) = + echo("Downloaded ", progress, " of ", total) + echo("Current rate: ", speed div 1000, "kb/s") + client.onProgressChanged = onProgressChanged + discard client.getContent("http://speedtest-ams2.digitalocean.com/100mb.test") + + client.close() + + # Timeout test. + client = newHttpClient(timeout = 1) + try: + resp = client.request("http://example.com/") + doAssert false, "TimeoutError should have been raised." + except TimeoutError: + discard + except: + doAssert false, "TimeoutError should have been raised." + +syncTest() + +waitFor(asyncTest()) diff --git a/tests/stdlib/tmarshal.nim b/tests/stdlib/tmarshal.nim index 4c0c10360..b05cb5a10 100644 --- a/tests/stdlib/tmarshal.nim +++ b/tests/stdlib/tmarshal.nim @@ -1,5 +1,7 @@ discard """ - output: '''{"age": 12, "name": "Cletus"}''' + output: '''{"age": 12, "bio": "\u042F Cletus", "blob": [65, 66, 67, 128], "name": "Cletus"} +true +true''' """ import marshal @@ -72,6 +74,12 @@ type Person = object of Entity age: int + bio: string + blob: string -var instance1 = Person(name: "Cletus", age: 12) +var instance1 = Person(name: "Cletus", age: 12, + bio: "Я Cletus", + blob: "ABC\x80") echo($$instance1) +echo(to[Person]($$instance1).bio == instance1.bio) +echo(to[Person]($$instance1).blob == instance1.blob) diff --git a/tests/stdlib/tnet_ll.nim b/tests/stdlib/tnet_ll.nim index 4d4df7c13..2ac272fd1 100644 --- a/tests/stdlib/tnet_ll.nim +++ b/tests/stdlib/tnet_ll.nim @@ -1,5 +1,8 @@ discard """ action: run + output: ''' +[Suite] inet_ntop tests +''' """ when defined(windows): diff --git a/tests/stdlib/tparseuints.nim b/tests/stdlib/tparseuints.nim index 5be3bcbd0..6b228d933 100644 --- a/tests/stdlib/tparseuints.nim +++ b/tests/stdlib/tparseuints.nim @@ -1,5 +1,7 @@ discard """ action: run + output: ''' +[Suite] parseutils''' """ import unittest, strutils diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index ac37196fb..3ab287c4e 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -39,55 +39,49 @@ doAssert t4.format("M MM MMM MMMM") == "10 10 Oct October" doAssert((t4 - initInterval(years = 2)).format("yyyy") == "1995") doAssert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") -var s = "Tuesday at 09:04am on Dec 15, 2015" -var f = "dddd at hh:mmtt on MMM d, yyyy" -doAssert($s.parse(f) == "Tue Dec 15 09:04:00 2015") +proc parseTest(s, f, sExpected: string, ydExpected: int) = + let parsed = s.parse(f) + doAssert($parsed == sExpected) + doAssert(parsed.yearday == ydExpected) +proc parseTestTimeOnly(s, f, sExpected: string) = + doAssert(sExpected in $s.parse(f)) + +parseTest("Tuesday at 09:04am on Dec 15, 2015", + "dddd at hh:mmtt on MMM d, yyyy", "Tue Dec 15 09:04:00 2015", 348) # ANSIC = "Mon Jan _2 15:04:05 2006" -s = "Thu Jan 12 15:04:05 2006" -f = "ddd MMM dd HH:mm:ss yyyy" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Thu Jan 12 15:04:05 2006", "ddd MMM dd HH:mm:ss yyyy", + "Thu Jan 12 15:04:05 2006", 11) # UnixDate = "Mon Jan _2 15:04:05 MST 2006" -s = "Thu Jan 12 15:04:05 MST 2006" -f = "ddd MMM dd HH:mm:ss ZZZ yyyy" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Thu Jan 12 15:04:05 MST 2006", "ddd MMM dd HH:mm:ss ZZZ yyyy", + "Thu Jan 12 15:04:05 2006", 11) # RubyDate = "Mon Jan 02 15:04:05 -0700 2006" -s = "Thu Jan 12 15:04:05 -07:00 2006" -f = "ddd MMM dd HH:mm:ss zzz yyyy" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Mon Feb 29 15:04:05 -07:00 2016", "ddd MMM dd HH:mm:ss zzz yyyy", + "Mon Feb 29 15:04:05 2016", 59) # leap day # RFC822 = "02 Jan 06 15:04 MST" -s = "12 Jan 16 15:04 MST" -f = "dd MMM yy HH:mm ZZZ" -doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +parseTest("12 Jan 16 15:04 MST", "dd MMM yy HH:mm ZZZ", + "Tue Jan 12 15:04:00 2016", 11) # RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone -s = "12 Jan 16 15:04 -07:00" -f = "dd MMM yy HH:mm zzz" -doAssert($s.parse(f) == "Tue Jan 12 15:04:00 2016") +parseTest("01 Mar 16 15:04 -07:00", "dd MMM yy HH:mm zzz", + "Tue Mar 1 15:04:00 2016", 60) # day after february in leap year # RFC850 = "Monday, 02-Jan-06 15:04:05 MST" -s = "Monday, 12-Jan-06 15:04:05 MST" -f = "dddd, dd-MMM-yy HH:mm:ss ZZZ" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Monday, 12-Jan-06 15:04:05 MST", "dddd, dd-MMM-yy HH:mm:ss ZZZ", + "Thu Jan 12 15:04:05 2006", 11) # RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" -s = "Thu, 12 Jan 2006 15:04:05 MST" -f = "ddd, dd MMM yyyy HH:mm:ss ZZZ" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Sun, 01 Mar 2015 15:04:05 MST", "ddd, dd MMM yyyy HH:mm:ss ZZZ", + "Sun Mar 1 15:04:05 2015", 59) # day after february in non-leap year # RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone -s = "Thu, 12 Jan 2006 15:04:05 -07:00" -f = "ddd, dd MMM yyyy HH:mm:ss zzz" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("Thu, 12 Jan 2006 15:04:05 -07:00", "ddd, dd MMM yyyy HH:mm:ss zzz", + "Thu Jan 12 15:04:05 2006", 11) # RFC3339 = "2006-01-02T15:04:05Z07:00" -s = "2006-01-12T15:04:05Z-07:00" -f = "yyyy-MM-ddTHH:mm:ssZzzz" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") -f = "yyyy-MM-dd'T'HH:mm:ss'Z'zzz" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("2006-01-12T15:04:05Z-07:00", "yyyy-MM-ddTHH:mm:ssZzzz", + "Thu Jan 12 15:04:05 2006", 11) +parseTest("2006-01-12T15:04:05Z-07:00", "yyyy-MM-dd'T'HH:mm:ss'Z'zzz", + "Thu Jan 12 15:04:05 2006", 11) # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" -s = "2006-01-12T15:04:05.999999999Z-07:00" -f = "yyyy-MM-ddTHH:mm:ss.999999999Zzzz" -doAssert($s.parse(f) == "Thu Jan 12 15:04:05 2006") +parseTest("2006-01-12T15:04:05.999999999Z-07:00", + "yyyy-MM-ddTHH:mm:ss.999999999Zzzz", "Thu Jan 12 15:04:05 2006", 11) # Kitchen = "3:04PM" -s = "3:04PM" -f = "h:mmtt" -doAssert "15:04:00" in $s.parse(f) +parseTestTimeOnly("3:04PM", "h:mmtt", "15:04:00") #when not defined(testing): # echo "Kitchen: " & $s.parse(f) # var ti = timeToTimeInfo(getTime()) diff --git a/tests/template/ttemp_in_varargs.nim b/tests/template/ttemp_in_varargs.nim new file mode 100644 index 000000000..be78e6ef2 --- /dev/null +++ b/tests/template/ttemp_in_varargs.nim @@ -0,0 +1,9 @@ +discard """ + output: '''a''' +""" + +# bug #4292 + +template foo(s: string): string = s +proc variadicProc*(v: varargs[string, foo]) = echo v[0] +variadicProc("a") diff --git a/tests/testament/backend.nim b/tests/testament/backend.nim index 671b5c8b7..8f0961566 100644 --- a/tests/testament/backend.nim +++ b/tests/testament/backend.nim @@ -59,6 +59,7 @@ var thisMachine: MachineId thisCommit: CommitId +{.experimental.} proc `()`(cmd: string{lit}): string = cmd.execProcess.string.strip proc getMachine*(db: DbConn): MachineId = diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index e534cc161..3ed2f2196 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -24,7 +24,7 @@ proc delNimCache() = echo "[Warning] could not delete: ", nimcacheDir proc runRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = + template test(filename: untyped) = testSpec r, makeTest(rodfilesDir / filename, options, cat, actionRun) delNimCache() @@ -46,18 +46,19 @@ proc runRodFiles(r: var TResults, cat: Category, options: string) = test "deada2" delNimCache() - # test method generation: - test "bmethods" - test "bmethods2" - delNimCache() + when false: + # test method generation: + test "bmethods" + test "bmethods2" + delNimCache() - # test generics: - test "tgeneric1" - test "tgeneric2" - delNimCache() + # test generics: + test "tgeneric1" + test "tgeneric2" + delNimCache() proc compileRodFiles(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = + template test(filename: untyped) = testSpec r, makeTest(rodfilesDir / filename, options, cat) delNimCache() @@ -114,20 +115,20 @@ proc dllTests(r: var TResults, cat: Category, options: string) = # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = - template testWithoutMs(filename: expr): stmt = + template testWithoutMs(filename: untyped) = testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release", cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release -d:useRealtimeGC", cat, actionRun) - template testWithoutBoehm(filename: expr): stmt = + template testWithoutBoehm(filename: untyped) = testWithoutMs filename testSpec r, makeTest("tests/gc" / filename, options & " --gc:markAndSweep", cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & " -d:release --gc:markAndSweep", cat, actionRun) - template test(filename: expr): stmt = + template test(filename: untyped) = testWithoutBoehm filename when not defined(windows): # AR: cannot find any boehm.dll on the net, right now, so disabled @@ -173,7 +174,7 @@ proc longGCTests(r: var TResults, cat: Category, options: string) = # ------------------------- threading tests ----------------------------------- proc threadTests(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = + template test(filename: untyped) = testSpec r, makeTest("tests/threads" / filename, options, cat, actionRun) testSpec r, makeTest("tests/threads" / filename, options & " -d:release", cat, actionRun) @@ -201,6 +202,14 @@ proc ioTests(r: var TResults, cat: Category, options: string) = testSpec c, makeTest("tests/system/helpers/readall_echo", options, cat) testSpec r, makeTest("tests/system/io", options, cat) +# ------------------------- async tests --------------------------------------- +proc asyncTests(r: var TResults, cat: Category, options: string) = + template test(filename: untyped) = + testSpec r, makeTest(filename, options, cat) + testSpec r, makeTest(filename, options & " -d:upcoming", cat) + for t in os.walkFiles("tests/async/t*.nim"): + test(t) + # ------------------------- debugger tests ------------------------------------ proc debuggerTests(r: var TResults, cat: Category, options: string) = @@ -209,7 +218,7 @@ proc debuggerTests(r: var TResults, cat: Category, options: string) = # ------------------------- JS tests ------------------------------------------ proc jsTests(r: var TResults, cat: Category, options: string) = - template test(filename: expr): stmt = + template test(filename: untyped) = testSpec r, makeTest(filename, options & " -d:nodejs", cat, actionRun, targetJS) testSpec r, makeTest(filename, options & " -d:nodejs -d:release", cat, @@ -220,14 +229,15 @@ proc jsTests(r: var TResults, cat: Category, options: string) = for testfile in ["exception/texceptions", "exception/texcpt1", "exception/texcsub", "exception/tfinally", "exception/tfinally2", "exception/tfinally3", + "exception/tunhandledexc", "actiontable/tactiontable", "method/tmultim1", "method/tmultim3", "method/tmultim4", "varres/tvarres0", "varres/tvarres3", "varres/tvarres4", "varres/tvartup", "misc/tints", "misc/tunsignedinc"]: test "tests/" & testfile & ".nim" - for testfile in ["pure/strutils", "pure/json", "pure/random", "pure/times"]: - test "lib/" & testfile & ".nim" + for testfile in ["strutils", "json", "random", "times", "logging"]: + test "lib/pure/" & testfile & ".nim" # ------------------------- manyloc ------------------------------------------- #proc runSpecialTests(r: var TResults, options: string) = @@ -369,9 +379,8 @@ proc `&?.`(a, b: string): string = proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: string = "t*.nim") = case cat.string.normalize of "rodfiles": - discard # Disabled for now - #compileRodFiles(r, cat, options) - #runRodFiles(r, cat, options) + when false: compileRodFiles(r, cat, options) + runRodFiles(r, cat, options) of "js": # XXX JS doesn't need to be special anymore jsTests(r, cat, options) @@ -389,6 +398,8 @@ proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: threadTests r, cat, options & " --threads:on" of "io": ioTests r, cat, options + of "async": + asyncTests r, cat, options of "lib": testStdlib(r, "lib/pure/*.nim", options, cat) testStdlib(r, "lib/packages/docutils/highlite", options, cat) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 83e59a6c1..74ac58927 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -334,6 +334,11 @@ proc testSpec(r: var TResults, test: TTest) = let exeCmd = (if isJsTarget: nodejs & " " else: "") & exeFile var (buf, exitCode) = execCmdEx(exeCmd, options = {poStdErrToStdOut}) + + # Treat all failure codes from nodejs as 1. Older versions of nodejs used + # to return other codes, but for us it is sufficient to know that it's not 0. + if exitCode != 0: exitCode = 1 + let bufB = if expected.sortoutput: makeDeterministic(strip(buf.string)) else: strip(buf.string) let expectedOut = strip(expected.outp) diff --git a/todo.txt b/todo.txt index 106f2bb34..21bec6e27 100644 --- a/todo.txt +++ b/todo.txt @@ -53,7 +53,6 @@ Bugs - VM: ptr/ref T cannot work in general - blocks can "export" an identifier but the CCG generates {} for them ... - ConcreteTypes in a 'case' means we don't check for duplicated case branches -- BUG: echo with template `$`*(info: TLineInfo): expr = toFileLineCol(info) GC diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim new file mode 100644 index 000000000..79a0e7482 --- /dev/null +++ b/tools/dochack/dochack.nim @@ -0,0 +1,296 @@ + + +import karax + +proc findNodeWith(x: Element; tag, content: cstring): Element = + if x.nodeName == tag and x.textContent == content: + return x + for i in 0..<x.len: + let it = x[i] + let y = findNodeWith(it, tag, content) + if y != nil: return y + return nil + +proc clone(e: Element): Element {.importcpp: "#.cloneNode(true)", nodecl.} +proc parent(e: Element): Element {.importcpp: "#.parentNode", nodecl.} +proc markElement(x: Element) {.importcpp: "#.__karaxMarker__ = true", nodecl.} +proc isMarked(x: Element): bool {. + importcpp: "#.hasOwnProperty('__karaxMarker__')", nodecl.} +proc title(x: Element): cstring {.importcpp: "#.title", nodecl.} + +proc sort[T](x: var openArray[T]; cmp: proc(a, b: T): int) {.importcpp: + "#.sort(#)", nodecl.} + +proc parentWith(x: Element; tag: cstring): Element = + result = x.parent + while result.nodeName != tag: + result = result.parent + if result == nil: return nil + +proc extractItems(x: Element; items: var seq[Element]) = + if x == nil: return + if x.nodeName == "A": + items.add x + else: + for i in 0..<x.len: + let it = x[i] + extractItems(it, items) + +# HTML trees are so shitty we transform the TOC into a decent +# data-structure instead and work on that. +type + TocEntry = ref object + heading: Element + kids: seq[TocEntry] + sortId: int + doSort: bool + +proc extractItems(x: TocEntry; heading: cstring; + items: var seq[Element]) = + if x == nil: return + if x.heading != nil and x.heading.textContent == heading: + for i in 0..<x.kids.len: + items.add x.kids[i].heading + else: + for i in 0..<x.kids.len: + let it = x.kids[i] + extractItems(it, heading, items) + +proc toHtml(x: TocEntry; isRoot=false): Element = + if x == nil: return nil + if x.kids.len == 0: + if x.heading == nil: return nil + return x.heading.clone + result = tree("DIV") + if x.heading != nil and not isMarked(x.heading): + result.add x.heading.clone + let ul = tree("UL") + if isRoot: + ul.setClass("simple simple-toc") + else: + ul.setClass("simple") + if x.dosort: + x.kids.sort(proc(a, b: TocEntry): int = + if a.heading != nil and b.heading != nil: + let x = a.heading.textContent + let y = b.heading.textContent + if x < y: return -1 + if x > y: return 1 + return 0 + else: + # ensure sorting is stable: + return a.sortId - b.sortId + ) + for k in x.kids: + let y = toHtml(k) + if y != nil: + ul.add tree("LI", y) + if ul.len != 0: result.add ul + if result.len == 0: result = nil + +proc containsWord(a, b: cstring): bool {.asmNoStackFrame.} = + {.emit: """ + var escaped = `b`.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); + return new RegExp("\\b" + escaped + "\\b").test(`a`); + """.} + +proc isWhitespace(text: cstring): bool {.asmNoStackFrame.} = + {.emit: """ + return !/[^\s]/.test(`text`); + """.} + +proc isWhitespace(x: Element): bool = + x.nodeName == "#text" and x.textContent.isWhitespace or + x.nodeName == "#comment" + +proc toToc(x: Element; father: TocEntry) = + if x.nodeName == "UL": + let f = TocEntry(heading: nil, kids: @[], sortId: father.kids.len) + var i = 0 + while i < x.len: + var nxt = i+1 + while nxt < x.len and x[nxt].isWhitespace: + inc nxt + if nxt < x.len and x[i].nodeName == "LI" and x[i].len == 1 and + x[nxt].nodeName == "UL": + let e = TocEntry(heading: x[i][0], kids: @[], sortId: f.kids.len) + let it = x[nxt] + for j in 0..<it.len: + toToc(it[j], e) + f.kids.add e + i = nxt+1 + else: + toToc(x[i], f) + inc i + father.kids.add f + elif isWhitespace(x): + discard + elif x.nodeName == "LI": + var idx: seq[int] = @[] + for i in 0 ..< x.len: + if not x[i].isWhitespace: idx.add i + if idx.len == 2 and x[idx[1]].nodeName == "UL": + let e = TocEntry(heading: x[idx[0]], kids: @[], + sortId: father.kids.len) + let it = x[idx[1]] + for j in 0..<it.len: + toToc(it[j], e) + father.kids.add e + else: + for i in 0..<x.len: + toToc(x[i], father) + else: + father.kids.add TocEntry(heading: x, kids: @[], + sortId: father.kids.len) + +proc tocul(x: Element): Element = + # x is a 'ul' element + result = tree("UL") + for i in 0..<x.len: + let it = x[i] + if it.nodeName == "LI": + result.add it.clone + elif it.nodeName == "UL": + result.add tocul(it) + +proc getSection(toc: Element; name: cstring): Element = + let sec = findNodeWith(toc, "A", name) + if sec != nil: + result = sec.parentWith("LI") + +proc uncovered(x: TocEntry): TocEntry = + if x.kids.len == 0 and x.heading != nil: + return if not isMarked(x.heading): x else: nil + result = TocEntry(heading: x.heading, kids: @[], sortId: x.sortId, + doSort: x.doSort) + for i in 0..<x.kids.len: + let y = uncovered(x.kids[i]) + if y != nil: result.kids.add y + if result.kids.len == 0: result = nil + +proc mergeTocs(orig, news: TocEntry): TocEntry = + result = uncovered(orig) + if result == nil: + result = news + else: + for i in 0..<news.kids.len: + result.kids.add news.kids[i] + +proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = + var newStuff = TocEntry(heading: nil, kids: @[], doSort: true) + for t in types: + let c = TocEntry(heading: t.clone, kids: @[], doSort: true) + t.markElement() + for p in procs: + if not isMarked(p): + let xx = karax.getElementsByClass(p.parent, cstring"attachedType") + if xx.len == 1 and xx[0].textContent == t.textContent: + #kout(cstring"found ", p.nodeName) + let q = tree("A", text(p.title)) + q.setAttr("href", p.getAttribute("href")) + c.kids.add TocEntry(heading: q, kids: @[]) + p.markElement() + newStuff.kids.add c + result = mergeTocs(orig, newStuff) + +var alternative: Element + +proc togglevis(d: Element) = + asm """ + if (`d`.style.display == 'none') + `d`.style.display = 'inline'; + else + `d`.style.display = 'none'; + """ + +proc groupBy*(value: cstring) {.exportc.} = + let toc = getElementById("toc-list") + if alternative.isNil: + var tt = TocEntry(heading: nil, kids: @[]) + toToc(toc, tt) + tt = tt.kids[0] + + var types: seq[Element] = @[] + var procs: seq[Element] = @[] + + extractItems(tt, "Types", types) + extractItems(tt, "Procs", procs) + extractItems(tt, "Converters", procs) + extractItems(tt, "Methods", procs) + extractItems(tt, "Templates", procs) + extractItems(tt, "Macros", procs) + extractItems(tt, "Iterators", procs) + + let ntoc = buildToc(tt, types, procs) + let x = toHtml(ntoc, isRoot=true) + alternative = tree("DIV", x) + if value == "type": + replaceById("tocRoot", alternative) + else: + replaceById("tocRoot", tree("DIV")) + togglevis(getElementById"toc-list") + +var + db: seq[Element] + contents: seq[cstring] + +template normalize(x: cstring): cstring = x.toLower.replace("_", "") + +proc dosearch(value: cstring): Element = + if db.isNil: + var stuff: Element + {.emit: """ + var request = new XMLHttpRequest(); + request.open("GET", "theindex.html", false); + request.send(null); + + var doc = document.implementation.createHTMLDocument("theindex"); + doc.documentElement.innerHTML = request.responseText; + + //parser=new DOMParser(); + //doc=parser.parseFromString("<html></html>", "text/html"); + + `stuff` = doc.documentElement; + """.} + db = stuff.getElementsByClass"reference external" + contents = @[] + for ahref in db: + contents.add ahref.textContent.normalize + let ul = tree("UL") + result = tree("DIV") + result.setClass"search_results" + var matches: seq[(Element, int)] = @[] + let key = value.normalize + for i in 0..<db.len: + let c = contents[i] + if c.containsWord(key): + matches.add((db[i], -(30_000 - c.len))) + elif c.contains(key): + matches.add((db[i], c.len)) + matches.sort do (a, b: auto) -> int: + a[1] - b[1] + for i in 0..min(<matches.len, 19): + ul.add(tree("LI", matches[i][0])) + if ul.len == 0: + result.add tree("B", text"no search results") + else: + result.add tree("B", text"search results") + result.add ul + +var oldtoc: Element +var timer: Timeout + +proc search*() {.exportc.} = + proc wrapper() = + let elem = getElementById("searchInput") + let value = elem.value + if value != "": + if oldtoc.isNil: + oldtoc = getElementById("tocRoot") + let results = dosearch(value) + replaceById("tocRoot", results) + elif not oldtoc.isNil: + replaceById("tocRoot", oldtoc) + + if timer != nil: clearTimeout(timer) + timer = setTimeout(wrapper, 400) diff --git a/tools/dochack/karax.nim b/tools/dochack/karax.nim new file mode 100644 index 000000000..d9619992b --- /dev/null +++ b/tools/dochack/karax.nim @@ -0,0 +1,343 @@ +# Simple lib to write JS UIs + +import dom + +export dom.Element, dom.Event, dom.cloneNode, dom + +proc kout*[T](x: T) {.importc: "console.log", varargs.} + ## the preferred way of debugging karax applications. + +proc id*(e: Node): cstring {.importcpp: "#.id", nodecl.} +proc `id=`*(e: Node; x: cstring) {.importcpp: "#.id = #", nodecl.} +proc className*(e: Node): cstring {.importcpp: "#.className", nodecl.} +proc `className=`*(e: Node; v: cstring) {.importcpp: "#.className = #", nodecl.} + +proc value*(e: Element): cstring {.importcpp: "#.value", nodecl.} +proc `value=`*(e: Element; v: cstring) {.importcpp: "#.value = #", nodecl.} + +proc getElementsByClass*(e: Element; name: cstring): seq[Element] {.importcpp: "#.getElementsByClassName(#)", nodecl.} + +proc toLower*(x: cstring): cstring {. + importcpp: "#.toLowerCase()", nodecl.} +proc replace*(x: cstring; search, by: cstring): cstring {. + importcpp: "#.replace(#, #)", nodecl.} + +type + EventHandler* = proc(ev: Event) + EventHandlerId* = proc(ev: Event; id: int) + + Timeout* = ref object + +var document* {.importc.}: Document + +var + dorender: proc (): Element {.closure.} + drawTimeout: Timeout + currentTree: Element + +proc setRenderer*(renderer: proc (): Element) = + dorender = renderer + +proc setTimeout*(action: proc(); ms: int): Timeout {.importc, nodecl.} +proc clearTimeout*(t: Timeout) {.importc, nodecl.} +proc targetElem*(e: Event): Element = cast[Element](e.target) + +proc getElementById*(id: cstring): Element {.importc: "document.getElementById", nodecl.} + +proc getElementsByClassName*(cls: cstring): seq[Element] {.importc: + "document.getElementsByClassName", nodecl.} + +proc textContent*(e: Element): cstring {. + importcpp: "#.textContent", nodecl.} + +proc replaceById*(id: cstring; newTree: Node) = + let x = getElementById(id) + x.parentNode.replaceChild(newTree, x) + newTree.id = id + +proc equals(a, b: Node): bool = + if a.nodeType != b.nodeType: return false + if a.id != b.id: return false + if a.nodeName != b.nodeName: return false + if a.nodeType == TextNode: + if a.data != b.data: return false + elif a.childNodes.len != b.childNodes.len: + return false + if a.className != b.className: + # style differences are updated in place and we pretend + # it's still the same node + a.className = b.className + #return false + return true + +proc diffTree(parent, a, b: Node) = + if equals(a, b): + if a.nodeType != TextNode: + # we need to do this correctly in the presence of asyncronous + # DOM updates: + var i = 0 + while i < a.childNodes.len and a.childNodes.len == b.childNodes.len: + diffTree(a, a.childNodes[i], b.childNodes[i]) + inc i + elif parent == nil: + replaceById("ROOT", b) + else: + parent.replaceChild(b, a) + +proc dodraw() = + let newtree = dorender() + newtree.id = "ROOT" + if currentTree == nil: + currentTree = newtree + replaceById("ROOT", currentTree) + else: + diffTree(nil, currentTree, newtree) + +proc redraw*() = + # we buffer redraw requests: + if drawTimeout != nil: + clearTimeout(drawTimeout) + drawTimeout = setTimeout(dodraw, 30) + +proc tree*(tag: string; kids: varargs[Element]): Element = + result = document.createElement tag + for k in kids: + result.appendChild k + +proc tree*(tag: string; attrs: openarray[(string, string)]; + kids: varargs[Element]): Element = + result = tree(tag, kids) + for a in attrs: result.setAttribute(a[0], a[1]) + +proc text*(s: string): Element = cast[Element](document.createTextNode(s)) +proc text*(s: cstring): Element = cast[Element](document.createTextNode(s)) +proc add*(parent, kid: Element) = + if parent.nodeName == "TR" and (kid.nodeName == "TD" or kid.nodeName == "TH"): + let k = document.createElement("TD") + appendChild(k, kid) + appendChild(parent, k) + else: + appendChild(parent, kid) + +proc len*(x: Element): int {.importcpp: "#.childNodes.length".} +proc `[]`*(x: Element; idx: int): Element {.importcpp: "#.childNodes[#]".} + +proc isInt*(s: cstring): bool {.asmNoStackFrame.} = + asm """ + return s.match(/^[0-9]+$/); + """ + +var + linkCounter: int + +proc link*(id: int): Element = + result = document.createElement("a") + result.setAttribute("href", "#") + inc linkCounter + result.setAttribute("id", $linkCounter & ":" & $id) + +proc link*(action: EventHandler): Element = + result = document.createElement("a") + result.setAttribute("href", "#") + addEventListener(result, "click", action) + +proc parseInt*(s: cstring): int {.importc, nodecl.} +proc parseFloat*(s: cstring): float {.importc, nodecl.} +proc split*(s, sep: cstring): seq[cstring] {.importcpp, nodecl.} + +proc startsWith*(a, b: cstring): bool {.importcpp: "startsWith", nodecl.} +proc contains*(a, b: cstring): bool {.importcpp: "(#.indexOf(#)>=0)", nodecl.} +proc substr*(s: cstring; start: int): cstring {.importcpp: "substr", nodecl.} +proc substr*(s: cstring; start, length: int): cstring {.importcpp: "substr", nodecl.} + +#proc len*(s: cstring): int {.importcpp: "#.length", nodecl.} +proc `&`*(a, b: cstring): cstring {.importcpp: "(# + #)", nodecl.} +proc toCstr*(s: int): cstring {.importcpp: "((#)+'')", nodecl.} + +proc suffix*(s, prefix: cstring): cstring = + if s.startsWith(prefix): + result = s.substr(prefix.len) + else: + kout(cstring"bug! " & s & cstring" does not start with " & prefix) + +proc valueAsInt*(e: Element): int = parseInt(e.value) +proc suffixAsInt*(s, prefix: cstring): int = parseInt(suffix(s, prefix)) + +proc scrollTop*(e: Element): int {.importcpp: "#.scrollTop", nodecl.} +proc offsetHeight*(e: Element): int {.importcpp: "#.offsetHeight", nodecl.} +proc offsetTop*(e: Element): int {.importcpp: "#.offsetTop", nodecl.} + +template onImpl(s) {.dirty} = + proc wrapper(ev: Event) = + action(ev) + redraw() + addEventListener(e, s, wrapper) + +proc setOnclick*(e: Element; action: proc(ev: Event)) = + onImpl "click" + +proc setOnclick*(e: Element; action: proc(ev: Event; id: int)) = + proc wrapper(ev: Event) = + let id = ev.target.id + let a = id.split(":") + if a.len == 2: + action(ev, parseInt(a[1])) + redraw() + else: + kout(cstring("cannot deal with id "), id) + addEventListener(e, "click", wrapper) + +proc setOnfocuslost*(e: Element; action: EventHandler) = + onImpl "blur" + +proc setOnchanged*(e: Element; action: EventHandler) = + onImpl "change" + +proc setOnscroll*(e: Element; action: EventHandler) = + onImpl "scroll" + +proc select*(choices: openarray[string]): Element = + result = document.createElement("select") + var i = 0 + for c in choices: + result.add tree("option", [("value", $i)], text(c)) + inc i + +proc select*(choices: openarray[(int, string)]): Element = + result = document.createElement("select") + for c in choices: + result.add tree("option", [("value", $c[0])], text(c[1])) + +var radioCounter: int + +proc radio*(choices: openarray[(int, string)]): Element = + result = document.createElement("fieldset") + var i = 0 + inc radioCounter + for c in choices: + let id = "radio_" & c[1] & $i + var kid = tree("input", [("type", "radio"), + ("id", id), ("name", "radio" & $radioCounter), + ("value", $c[0])]) + if i == 0: + kid.setAttribute("checked", "checked") + var lab = tree("label", [("for", id)], text(c[1])) + kid.add lab + result.add kid + inc i + +proc tag*(name: string; id="", class=""): Element = + result = document.createElement(name) + if id.len > 0: + result.setAttribute("id", id) + if class.len > 0: + result.setAttribute("class", class) + +proc tdiv*(id="", class=""): Element = tag("div", id, class) +proc span*(id="", class=""): Element = tag("span", id, class) + +proc th*(s: string): Element = + result = tag("th") + result.add text(s) + +proc td*(s: string): Element = + result = tag("td") + result.add text(s) + +proc td*(s: Element): Element = + result = tag("td") + result.add s + +proc td*(class: string; s: Element): Element = + result = tag("td") + result.add s + result.setAttribute("class", class) + +proc table*(class="", kids: varargs[Element]): Element = + result = tag("table", "", class) + for k in kids: result.add k + +proc tr*(kids: varargs[Element]): Element = + result = tag("tr") + for k in kids: + if k.nodeName == "TD" or k.nodeName == "TH": + result.add k + else: + result.add td(k) + +proc setClass*(e: Element; value: string) = + e.setAttribute("class", value) + +proc setAttr*(e: Element; key, value: cstring) = + e.setAttribute(key, value) + +proc getAttr*(e: Element; key: cstring): cstring {. + importcpp: "#.getAttribute(#)", nodecl.} + +proc realtimeInput*(id, val: string; changed: proc(value: cstring)): Element = + let oldElem = getElementById(id) + #if oldElem != nil: return oldElem + let newVal = if oldElem.isNil: val else: $oldElem.value + var timer: Timeout + proc wrapper() = + changed(getElementById(id).value) + redraw() + proc onkeyup(ev: Event) = + if timer != nil: clearTimeout(timer) + timer = setTimeout(wrapper, 400) + result = tree("input", [("type", "text"), + ("value", newVal), + ("id", id)]) + result.addEventListener("keyup", onkeyup) + +proc ajax(meth, url: cstring; headers: openarray[(string, string)]; + data: cstring; + cont: proc (httpStatus: int; response: cstring)) = + proc setRequestHeader(a, b: cstring) {.importc: "ajax.setRequestHeader".} + {.emit: """ + var ajax = new XMLHttpRequest(); + ajax.open(`meth`,`url`,true);""".} + for a, b in items(headers): + setRequestHeader(a, b) + {.emit: """ + ajax.onreadystatechange = function(){ + if(this.readyState == 4){ + if(this.status == 200){ + `cont`(this.status, this.responseText); + } else { + `cont`(this.status, this.statusText); + } + } + } + ajax.send(`data`); + """.} + +proc ajaxPut*(url: string; headers: openarray[(string, string)]; + data: cstring; + cont: proc (httpStatus: int, response: cstring)) = + ajax("PUT", url, headers, data, cont) + +proc ajaxGet*(url: string; headers: openarray[(string, string)]; + cont: proc (httpStatus: int, response: cstring)) = + ajax("GET", url, headers, nil, cont) + +{.push stackTrace:off.} + +proc setupErrorHandler*(useAlert=false) = + ## Installs an error handler that transforms native JS unhandled + ## exceptions into Nim based stack traces. If `useAlert` is false, + ## the error message it put into the console, otherwise `alert` + ## is called. + proc stackTraceAsCstring(): cstring = cstring(getStackTrace()) + {.emit: """ + window.onerror = function(msg, url, line, col, error) { + var x = "Error: " + msg + "\n" + `stackTraceAsCstring`() + if (`useAlert`) + alert(x); + else + console.log(x); + var suppressErrorAlert = true; + return suppressErrorAlert; + };""".} + +{.pop.} diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index e93168847..8dff722ec 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -11,7 +11,7 @@ import os, strutils, parseopt, pegs, re, terminal const - Version = "1.0" + Version = "1.1" Usage = "nimgrep - Nim Grep Utility Version " & Version & """ (c) 2012 Andreas Rumpf @@ -135,7 +135,7 @@ proc processFile(filename: string) = if optVerbose in options: stdout.writeLine(filename) stdout.flushFile() - var pegp: TPeg + var pegp: Peg var rep: Regex var result: string @@ -161,7 +161,7 @@ proc processFile(filename: string) = t = findBounds(buffer, pegp, matches, i) else: t = findBounds(buffer, rep, matches, i) - if t.first <= 0: break + if t.first < 0: break inc(line, countLines(buffer, i, t.first-1)) var wholeMatch = buffer.substr(t.first, t.last) @@ -213,7 +213,7 @@ proc hasRightExt(filename: string, exts: seq[string]): bool = if os.cmpPaths(x, y) == 0: return true proc styleInsensitive(s: string): string = - template addx: stmt = + template addx = result.add(s[i]) inc(i) result = "" diff --git a/tools/niminst/buildsh.tmpl b/tools/niminst/buildsh.tmpl index 13dfe5226..220ecdb7f 100644 --- a/tools/niminst/buildsh.tmpl +++ b/tools/niminst/buildsh.tmpl @@ -92,6 +92,7 @@ case $uos in ;; *haiku* ) myos="haiku" + LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" ;; *) echo 2>&1 "Error: unknown operating system: $uos" diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index 4c8dfcddf..b63849a10 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -447,7 +447,8 @@ proc readCFiles(c: var ConfigData, osA, cpuA: int) = # HACK: we conditionally add ``-lm -ldl``, so remove them from the # linker flags: c.linker.flags = c.linker.flags.replaceWord("-lm").replaceWord( - "-ldl").strip + "-ldl").replaceWord("-lroot").replaceWord( + "-lnetwork").strip else: if cmpIgnoreStyle(k.key, "libpath") == 0: c.libpath = k.value @@ -616,9 +617,8 @@ when haveZipLib: else: quit("Cannot open for writing: " & n) -proc xzDist(c: var ConfigData) = +proc xzDist(c: var ConfigData; windowsZip=false) = let proj = toLower(c.name) & "-" & c.version - var n = "$#.tar.xz" % proj let tmpDir = if c.outdir.len == 0: "build" else: c.outdir template processFile(z, dest, src) = @@ -635,15 +635,17 @@ proc xzDist(c: var ConfigData) = processFile(z, proj / makeFile, "build" / makeFile) processFile(z, proj / installShFile, installShFile) processFile(z, proj / deinstallShFile, deinstallShFile) - for f in walkFiles(c.libpath / "lib/*.h"): - processFile(z, proj / "c_code" / extractFilename(f), f) - for osA in 1..c.oses.len: - for cpuA in 1..c.cpus.len: - var dir = buildDir(osA, cpuA) - for k, f in walkDir("build" / dir): - if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) - - for cat in items({fcConfig..fcOther, fcUnix, fcNimble}): + if not windowsZip: + for f in walkFiles(c.libpath / "lib/*.h"): + processFile(z, proj / "c_code" / extractFilename(f), f) + for osA in 1..c.oses.len: + for cpuA in 1..c.cpus.len: + var dir = buildDir(osA, cpuA) + for k, f in walkDir("build" / dir): + if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) + + let osSpecific = if windowsZip: fcWindows else: fcUnix + for cat in items({fcConfig..fcOther, osSpecific, fcNimble}): echo("Current category: ", cat) for f in items(c.cat[cat]): processFile(z, proj / f, f) @@ -655,10 +657,15 @@ proc xzDist(c: var ConfigData) = let oldDir = getCurrentDir() setCurrentDir(tmpDir) try: - if execShellCmd("XZ_OPT=-9 gtar Jcf $1.tar.xz $1 --exclude=.DS_Store" % proj) != 0: - # try old 'tar' without --exclude feature: - if execShellCmd("XZ_OPT=-9 tar Jcf $1.tar.xz $1" % proj) != 0: + if windowsZip: + if execShellCmd("7z a -tzip $1.zip $1" % proj) != 0: echo("External program failed") + else: + if execShellCmd("XZ_OPT=-9 gtar Jcf $1.tar.xz $1 --exclude=.DS_Store" % + proj) != 0: + # try old 'tar' without --exclude feature: + if execShellCmd("XZ_OPT=-9 tar Jcf $1.tar.xz $1" % proj) != 0: + echo("External program failed") finally: setCurrentDir(oldDir) @@ -724,10 +731,7 @@ if actionCSource in c.actions: if actionScripts in c.actions: writeInstallScripts(c) if actionZip in c.actions: - when haveZipLib: - zipDist(c) - else: - quit("libzip is not installed") + xzDist(c, true) if actionXz in c.actions: xzDist(c) if actionDeb in c.actions: diff --git a/tools/niminst/nsis.tmpl b/tools/niminst/nsis.tmpl index 0897d36a8..639a01b6b 100644 --- a/tools/niminst/nsis.tmpl +++ b/tools/niminst/nsis.tmpl @@ -11,10 +11,6 @@ ; File Functions Header, used to get the current drive root. !include "FileFunc.nsh" - ; *Patched* Environment Variable Manipulation Header, used to add - ; tools to the user's PATH environment variable. - !include "EnvVarUpdate.nsh" - ;-------------------------------- ; Global variables and defines !define PRODUCT_NAME "?c.displayName" @@ -35,7 +31,7 @@ ; Default installation folder ; This is changed later (in .onInit) to the root directory, if possible. - InstallDir "$PROGRAMFILES?{when sizeof(int) == 8: "64" else: ""}\?{c.name}-?{c.version}" + InstallDir "$PROGRAMFILES?{when sizeof(int) == 8: "64" else: ""}\?{c.name}-?{c.version}\" ; Get installation folder from registry if available InstallDirRegKey HKCU "Software\c.name\c.version" "" @@ -44,7 +40,7 @@ RequestExecutionLevel user ; Allow installation to the root drive directory. - AllowRootDirInstall true + AllowRootDirInstall false ; Maximum compression! SetCompressor /SOLID /FINAL lzma @@ -156,10 +152,14 @@ ; Section for adding tools to the PATH variable Section "Setup Path Environment" PathSection - ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\mingw" - ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\mingw\bin" - ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\bin" - ${EnvVarUpdate} $R0 "PATH" "A" "HKCU" "$INSTDIR\dist\babel" + Push "$INSTDIR\bin" + Call AddToPath + Push "$INSTDIR\dist\mingw" + Call AddToPath + Push "$INSTDIR\dist\mingw\bin" + Call AddToPath + Push "$PROFILE\.nimble\bin" + Call AddToPath SectionEnd ; The downloadable sections. These sections are automatically generated by @@ -243,10 +243,14 @@ SetAutoClose true ; Remove entries from the PATH environment variable - ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\mingw" - ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\mingw\bin" - ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\bin" - ${un.EnvVarUpdate} $R0 "PATH" "R" "HKCU" "$INSTDIR\dist\babel" + Push "$INSTDIR\bin" + Call un.RemoveFromPath + Push "$INSTDIR\dist\mingw" + Call un.RemoveFromPath + Push "$INSTDIR\dist\mingw\bin" + Call un.RemoveFromPath + Push "$PROFILE\.nimble\bin" + Call un.RemoveFromPath SectionEnd ;-------------------------------- @@ -254,5 +258,188 @@ Function .onInit ${GetRoot} "$EXEDIR" $R0 - strCpy $INSTDIR "$R0\?{c.name}" + ;strCpy $INSTDIR "$R0\?{c.name}" FunctionEnd + + +;-------------------------------------------------------------------- +; Path functions +; +; Based on example from: +; http://nsis.sourceforge.net/Path_Manipulation +; +; Actually based on: +; https://www.smartmontools.org/browser/trunk/smartmontools/os_win32/installer.nsi#L636 + + +!include "WinMessages.nsh" + +; Registry Entry for environment (NT4,2000,XP) +; All users: +;!define Environ 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"' +; Current user only: +!define Environ 'HKCU "Environment"' + + +; AddToPath - Appends dir to PATH +; (does not work on Win9x/ME) +; +; Usage: +; Push "dir" +; Call AddToPath + +Function AddToPath + Exch $0 + Push $1 + Push $2 + Push $3 + Push $4 + + ; NSIS ReadRegStr returns empty string on string overflow + ; Native calls are used here to check actual length of PATH + + ; $4 = RegOpenKey(HKEY_CURRENT_USER, "Environment", &$3) + System::Call "advapi32::RegOpenKey(i 0x80000001, t'Environment', *i.r3) i.r4" + IntCmp $4 0 0 done done + ; $4 = RegQueryValueEx($3, "PATH", (DWORD*)0, (DWORD*)0, &$1, ($2=NSIS_MAX_STRLEN, &$2)) + ; RegCloseKey($3) + System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4" + System::Call "advapi32::RegCloseKey(i $3)" + + IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA + DetailPrint "AddToPath: original length $2 > ${NSIS_MAX_STRLEN}" + MessageBox MB_OK "PATH not updated, original length $2 > ${NSIS_MAX_STRLEN}" + Goto done + + IntCmp $4 0 +5 ; $4 != NO_ERROR + IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND + DetailPrint "AddToPath: unexpected error code $4" + Goto done + StrCpy $1 "" + + ; Check if already in PATH + Push "$1;" + Push "$0;" + Call StrStr + Pop $2 + StrCmp $2 "" 0 done + Push "$1;" + Push "$0\;" + Call StrStr + Pop $2 + StrCmp $2 "" 0 done + + ; Prevent NSIS string overflow + StrLen $2 $0 + StrLen $3 $1 + IntOp $2 $2 + $3 + IntOp $2 $2 + 2 ; $2 = strlen(dir) + strlen(PATH) + sizeof(";") + IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0 + DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}" + MessageBox MB_OK "PATH not updated, new length $2 > ${NSIS_MAX_STRLEN}." + Goto done + + ; Append dir to PATH + DetailPrint "Add to PATH: $0" + StrCpy $2 $1 1 -1 + StrCmp $2 ";" 0 +2 + StrCpy $1 $1 -1 ; remove trailing ';' + StrCmp $1 "" +2 ; no leading ';' + StrCpy $0 "$1;$0" + WriteRegExpandStr ${Environ} "PATH" $0 + SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 + +done: + Pop $4 + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + + +; RemoveFromPath - Removes dir from PATH +; +; Usage: +; Push "dir" +; Call RemoveFromPath + +Function un.RemoveFromPath + Exch $0 + Push $1 + Push $2 + Push $3 + Push $4 + Push $5 + Push $6 + + ReadRegStr $1 ${Environ} "PATH" + StrCpy $5 $1 1 -1 + StrCmp $5 ";" +2 + StrCpy $1 "$1;" ; ensure trailing ';' + Push $1 + Push "$0;" + Call un.StrStr + Pop $2 ; pos of our dir + StrCmp $2 "" done + + DetailPrint "Remove from PATH: $0" + StrLen $3 "$0;" + StrLen $4 $2 + StrCpy $5 $1 -$4 ; $5 is now the part before the path to remove + StrCpy $6 $2 "" $3 ; $6 is now the part after the path to remove + StrCpy $3 "$5$6" + StrCpy $5 $3 1 -1 + StrCmp $5 ";" 0 +2 + StrCpy $3 $3 -1 ; remove trailing ';' + WriteRegExpandStr ${Environ} "PATH" $3 + SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 + +done: + Pop $6 + Pop $5 + Pop $4 + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + + +; StrStr - find substring in a string +; +; Usage: +; Push "this is some string" +; Push "some" +; Call StrStr +; Pop $0 ; "some string" + +!macro StrStr un +Function ${un}StrStr + Exch $R1 ; $R1=substring, stack=[old$R1,string,...] + Exch ; stack=[string,old$R1,...] + Exch $R2 ; $R2=string, stack=[old$R2,old$R1,...] + Push $R3 + Push $R4 + Push $R5 + StrLen $R3 $R1 + StrCpy $R4 0 + ; $R1=substring, $R2=string, $R3=strlen(substring) + ; $R4=count, $R5=tmp + loop: + StrCpy $R5 $R2 $R3 $R4 + StrCmp $R5 $R1 done + StrCmp $R5 "" done + IntOp $R4 $R4 + 1 + Goto loop +done: + StrCpy $R1 $R2 "" $R4 + Pop $R5 + Pop $R4 + Pop $R3 + Pop $R2 + Exch $R1 ; $R1=old$R1, stack=[result,...] +FunctionEnd +!macroend +!insertmacro StrStr "" +!insertmacro StrStr "un." diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 4cf7020c2..65af67216 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -13,6 +13,8 @@ import from xmltree import escape +const gitRepo = "https://github.com/nim-lang/Nim" + type TKeyValPair = tuple[key, id, val: string] TConfigData = object of RootObj @@ -21,8 +23,6 @@ type authors, projectName, projectTitle, logo, infile, outdir, ticker: string vars: StringTableRef nimArgs: string - gitRepo: string - gitCommit: string quotations: Table[string, tuple[quote, author: string]] numProcessors: int # Set by parallelBuild:n, only works for values > 0. gaId: string # google analytics ID, nil means analytics are disabled @@ -59,8 +59,6 @@ proc initConfigData(c: var TConfigData) = c.logo = "" c.ticker = "" c.vars = newStringTable(modeStyleInsensitive) - c.gitRepo = "https://github.com/nim-lang/Nim/tree" - c.gitCommit = "master" c.numProcessors = countProcessors() # Attempts to obtain the git current commit. when false: @@ -94,16 +92,17 @@ Compile_options: rYearMonthDay = r"(\d{4})_(\d{2})_(\d{2})" rssUrl = "http://nim-lang.org/news.xml" rssNewsUrl = "http://nim-lang.org/news.html" - sponsors = "web/sponsors.csv" + activeSponsors = "web/sponsors.csv" + inactiveSponsors = "web/inactive_sponsors.csv" validAnchorCharacters = Letters + Digits -macro id(e: expr): expr {.immediate.} = +macro id(e: untyped): untyped = ## generates the rss xml ``id`` element. let e = callsite() result = xmlCheckedTag(e, "id") -macro updated(e: expr): expr {.immediate.} = +macro updated(e: varargs[untyped]): untyped = ## generates the rss xml ``updated`` element. let e = callsite() result = xmlCheckedTag(e, "updated") @@ -114,12 +113,12 @@ proc updatedDate(year, month, day: string): string = repeat("0", 2 - len(month)) & month, repeat("0", 2 - len(day)) & day]) -macro entry(e: expr): expr {.immediate.} = +macro entry(e: varargs[untyped]): untyped = ## generates the rss xml ``entry`` element. let e = callsite() result = xmlCheckedTag(e, "entry") -macro content(e: expr): expr {.immediate.} = +macro content(e: varargs[untyped]): untyped = ## generates the rss xml ``content`` element. let e = callsite() result = xmlCheckedTag(e, "content", reqAttr = "type") @@ -244,11 +243,6 @@ proc parseIniFile(c: var TConfigData) = c.projectName = changeFileExt(extractFilename(c.infile), "") if c.outdir.len == 0: c.outdir = splitFile(c.infile).dir - # Ugly hack to override git command output when building private repo. - if c.vars.hasKey("githash"): - let githash = c.vars["githash"].strip - if githash.len > 0: - c.gitCommit = githash # ------------------- main ---------------------------------------------------- @@ -302,18 +296,18 @@ proc buildDoc(c: var TConfigData, destPath: string) = commands = newSeq[string](len(c.doc) + len(c.srcdoc) + len(c.srcdoc2)) i = 0 for d in items(c.doc): - commands[i] = findNim() & " rst2html $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + commands[i] = findNim() & " rst2html $# --git.url:$# -o:$# --index:on $#" % + [c.nimArgs, gitRepo, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc for d in items(c.srcdoc): - commands[i] = findNim() & " doc $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + commands[i] = findNim() & " doc $# --git.url:$# -o:$# --index:on $#" % + [c.nimArgs, gitRepo, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc for d in items(c.srcdoc2): - commands[i] = findNim() & " doc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + commands[i] = findNim() & " doc2 $# --git.url:$# -o:$# --index:on $#" % + [c.nimArgs, gitRepo, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc @@ -345,8 +339,8 @@ proc buildAddDoc(c: var TConfigData, destPath: string) = # build additional documentation (without the index): var commands = newSeq[string](c.webdoc.len) for i, doc in pairs(c.webdoc): - commands[i] = findNim() & " doc2 $# --docSeeSrcUrl:$#/$# -o:$# $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, + commands[i] = findNim() & " doc2 $# --git.url:$# -o:$# $#" % + [c.nimArgs, gitRepo, destPath / changeFileExt(splitFile(doc).name, "html"), doc] mexec(commands, c.numProcessors) @@ -446,8 +440,9 @@ proc readSponsors(sponsorsFile: string): seq[Sponsor] = since: parser.row[5], level: parser.row[6].parseInt)) parser.close() -proc buildSponsors(c: var TConfigData, sponsorsFile: string, outputDir: string) = - let sponsors = generateSponsors(readSponsors(sponsorsFile)) +proc buildSponsors(c: var TConfigData, outputDir: string) = + let sponsors = generateSponsorsPage(readSponsors(activeSponsors), + readSponsors(inactiveSponsors)) let outFile = outputDir / "sponsors.html" var f: File if open(f, outFile, fmWrite): @@ -500,7 +495,7 @@ proc buildWebsite(c: var TConfigData) = buildPage(c, file, if file == "question": "FAQ" else: file, rss) copyDir("web/assets", "web/upload/assets") buildNewsRss(c, "web/upload") - buildSponsors(c, sponsors, "web/upload") + buildSponsors(c, "web/upload") buildNews(c, "web/news", "web/upload/news") proc main(c: var TConfigData) = @@ -518,8 +513,8 @@ proc json2(c: var TConfigData) = var i = 0 for d in items(c.srcdoc2): createDir(destPath / splitFile(d).dir) - commands[i] = findNim() & " jsondoc2 $# --docSeeSrcUrl:$#/$#/$# -o:$# --index:on $#" % - [c.nimArgs, c.gitRepo, c.gitCommit, d.pathPart, + commands[i] = findNim() & " jsondoc2 $# --git.url:$# -o:$# --index:on $#" % + [c.nimArgs, gitRepo, destPath / changeFileExt(d, "json"), d] i.inc diff --git a/tools/website.tmpl b/tools/website.tmpl index cf2c72a60..2801fea96 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -66,7 +66,7 @@ </a> </div> <div id="slide1" class="niaslide"> - <a href="news.html#Z2016-01-27-nim-in-action-is-now-available"> + <a href="news/2016_01_27_nim_in_action_is_now_available.html"> <img src="${rootDir}assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!"/> </a> </div> @@ -180,8 +180,8 @@ runForever() <div id="foot-links"> <div> <h4>Documentation</h4> - <a href="documentation.html">Stable Documentation</a> - <a href="learn.html">Learning Resources</a> + <a href="${rootDir}documentation.html">Stable Documentation</a> + <a href="${rootDir}learn.html">Learning Resources</a> <!-- <a href="">Development Documentation</a> --> <a href="https://github.com/nim-lang/nim">Issues & Requests</a> </div> @@ -218,14 +218,9 @@ runForever() </html> #end proc # +# #proc generateSponsors(sponsors: seq[Sponsor]): string = #result = "" -<h1 id="our-current-sponsors">Our Current Sponsors</h1> -<p>This page lists the companies and individuals that are, very kindly, contributing a -monthly amount to help sustain Nim's development. For more details take a -look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> -<p class="lastUpdate">Last updated: ${getTime().getGMTime().format("dd/MM/yyyy")}</p> -<dl> #for sponsor in sponsors: <dt class="level-${sponsor.level}"> #if sponsor.url.len > 0: @@ -248,6 +243,24 @@ look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campa Donated $$${sponsor.allTime} in total since ${sponsor.since} </dd> #end for +#end proc +#proc generateSponsorsPage(activeSponsors, inactiveSponsors: seq[Sponsor]): string = +#result = "" +<h1 id="our-current-sponsors">Our Current Sponsors</h1> +<p>This section lists the companies and individuals that are, very kindly, contributing a +monthly amount to help sustain Nim's development. For more details take a +look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> +<p class="lastUpdate">Last updated: ${getTime().getGMTime().format("dd/MM/yyyy")}</p> +<dl> +${generateSponsors(activeSponsors)} +</dl> +# +<h1 id="our-past-sponsors">Our Past Sponsors</h1> +<p>This section lists the companies and individuals that have contributed +money in the past to help sustain Nim's development. For more details take a +look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> +<dl> +${generateSponsors(inactiveSponsors)} </dl> # #end proc diff --git a/web/assets/news/images/0.15.0/doc_search.gif b/web/assets/news/images/0.15.0/doc_search.gif new file mode 100644 index 000000000..ac757b404 --- /dev/null +++ b/web/assets/news/images/0.15.0/doc_search.gif Binary files differdiff --git a/web/assets/news/images/0.15.0/doc_sort.gif b/web/assets/news/images/0.15.0/doc_sort.gif new file mode 100644 index 000000000..edd253c4a --- /dev/null +++ b/web/assets/news/images/0.15.0/doc_sort.gif Binary files differdiff --git a/web/assets/news/images/survey/10_needs.png b/web/assets/news/images/survey/10_needs.png new file mode 100644 index 000000000..67d568552 --- /dev/null +++ b/web/assets/news/images/survey/10_needs.png Binary files differdiff --git a/web/assets/news/images/survey/book.png b/web/assets/news/images/survey/book.png new file mode 100644 index 000000000..5bb418e63 --- /dev/null +++ b/web/assets/news/images/survey/book.png Binary files differdiff --git a/web/assets/news/images/survey/book_opinion.png b/web/assets/news/images/survey/book_opinion.png new file mode 100644 index 000000000..4e56ab26e --- /dev/null +++ b/web/assets/news/images/survey/book_opinion.png Binary files differdiff --git a/web/assets/news/images/survey/breakage.png b/web/assets/news/images/survey/breakage.png new file mode 100644 index 000000000..5eb4c5289 --- /dev/null +++ b/web/assets/news/images/survey/breakage.png Binary files differdiff --git a/web/assets/news/images/survey/difficulty_fixing_breakage.png b/web/assets/news/images/survey/difficulty_fixing_breakage.png new file mode 100644 index 000000000..022aa00ed --- /dev/null +++ b/web/assets/news/images/survey/difficulty_fixing_breakage.png Binary files differdiff --git a/web/assets/news/images/survey/domains.png b/web/assets/news/images/survey/domains.png new file mode 100644 index 000000000..50b1ed7ff --- /dev/null +++ b/web/assets/news/images/survey/domains.png Binary files differdiff --git a/web/assets/news/images/survey/ex_nim.png b/web/assets/news/images/survey/ex_nim.png new file mode 100644 index 000000000..50082ea8b --- /dev/null +++ b/web/assets/news/images/survey/ex_nim.png Binary files differdiff --git a/web/assets/news/images/survey/languages.png b/web/assets/news/images/survey/languages.png new file mode 100644 index 000000000..db35f9bd4 --- /dev/null +++ b/web/assets/news/images/survey/languages.png Binary files differdiff --git a/web/assets/news/images/survey/learning_resources.png b/web/assets/news/images/survey/learning_resources.png new file mode 100644 index 000000000..39f533ad0 --- /dev/null +++ b/web/assets/news/images/survey/learning_resources.png Binary files differdiff --git a/web/assets/news/images/survey/nim_appeal.png b/web/assets/news/images/survey/nim_appeal.png new file mode 100644 index 000000000..4f53e1447 --- /dev/null +++ b/web/assets/news/images/survey/nim_appeal.png Binary files differdiff --git a/web/assets/news/images/survey/nim_displeasing.png b/web/assets/news/images/survey/nim_displeasing.png new file mode 100644 index 000000000..b7232df04 --- /dev/null +++ b/web/assets/news/images/survey/nim_displeasing.png Binary files differdiff --git a/web/assets/news/images/survey/nim_domains.png b/web/assets/news/images/survey/nim_domains.png new file mode 100644 index 000000000..2d8fc6652 --- /dev/null +++ b/web/assets/news/images/survey/nim_domains.png Binary files differdiff --git a/web/assets/news/images/survey/nimble_opinion.png b/web/assets/news/images/survey/nimble_opinion.png new file mode 100644 index 000000000..3fe76326e --- /dev/null +++ b/web/assets/news/images/survey/nimble_opinion.png Binary files differdiff --git a/web/assets/news/images/survey/non_user.png b/web/assets/news/images/survey/non_user.png new file mode 100644 index 000000000..b5324b69c --- /dev/null +++ b/web/assets/news/images/survey/non_user.png Binary files differdiff --git a/web/assets/news/images/survey/planning_to_use_at_work.png b/web/assets/news/images/survey/planning_to_use_at_work.png index be3a50467..b3e2001c3 100644 --- a/web/assets/news/images/survey/planning_to_use_at_work.png +++ b/web/assets/news/images/survey/planning_to_use_at_work.png Binary files differdiff --git a/web/assets/news/images/survey/project_size_nim_rust.png b/web/assets/news/images/survey/project_size_nim_rust.png index 41e3ec8b1..4bc0e6b47 100644 --- a/web/assets/news/images/survey/project_size_nim_rust.png +++ b/web/assets/news/images/survey/project_size_nim_rust.png Binary files differdiff --git a/web/assets/news/images/survey/upgrades_broke_things.png b/web/assets/news/images/survey/upgrades_broke_things.png deleted file mode 100644 index 28a8ee3f0..000000000 --- a/web/assets/news/images/survey/upgrades_broke_things.png +++ /dev/null Binary files differdiff --git a/web/assets/style.css b/web/assets/style.css index af32fad07..2e166530d 100644 --- a/web/assets/style.css +++ b/web/assets/style.css @@ -516,12 +516,11 @@ pre .end { background:url("images/tabEnd.png") no-repeat left bottom; } padding:5px 30px; margin-bottom:20px; border:8px solid rgba(0,0,0,.8); - border-right-width:16px; + border-right-width:0; border-top-width:0; border-bottom-width:0; border-radius:3px; - background:rgba(0,0,0,0.1); - box-shadow:1px 3px 12px rgba(0,0,0,.4); } + background:rgba(0,0,0,0.1); } .standout h2 { margin-bottom:10px; padding-bottom:10px; border-bottom:1px dashed rgba(0,0,0,.8); } .standout li { margin:0 !important; padding-top:10px; border-top:1px dashed rgba(0,0,0,.2); } .standout ul { padding-bottom:5px; } @@ -611,6 +610,15 @@ p.lastUpdate { color: #6d6d6d; } +/* quotes */ + +blockquote { + padding: 0px 8px; + margin: 10px 0px; + border-left: 2px solid rgb(61, 61, 61); + color: rgb(109, 109, 109); +} + /* News articles */ .metadata { diff --git a/web/bountysource.nim b/web/bountysource.nim index a4f76417e..1d47cea56 100644 --- a/web/bountysource.nim +++ b/web/bountysource.nim @@ -29,12 +29,6 @@ proc newBountySource(team, token: string): BountySource = result.client.headers["Referer"] = "https://salt.bountysource.com/teams/nim/admin/supporters" result.client.headers["Origin"] = "https://salt.bountysource.com/" -proc getSupportLevels(self: BountySource): Future[JsonNode] {.async.} = - let response = await self.client.get(apiUrl & - "/support_levels?supporters_for_team=" & self.team) - doAssert response.status.startsWith($Http200) # TODO: There should be a == - return parseJson(response.body) - proc getSupporters(self: BountySource): Future[JsonNode] {.async.} = let response = await self.client.get(apiUrl & "/supporters?order=monthly&per_page=200&team_slug=" & self.team) @@ -47,26 +41,21 @@ proc getGithubUser(username: string): Future[JsonNode] {.async.} = if response.status.startsWith($Http200): return parseJson(response.body) else: + echo("Could not get Github user: ", username, ". ", response.status) return nil -proc processLevels(supportLevels: JsonNode) = - var before = supportLevels.elems.len - supportLevels.elems.keepIf( - item => item["status"].getStr == "active" and - item["owner"]["display_name"].getStr != "Anonymous" +proc processSupporters(supporters: JsonNode) = + var before = supporters.elems.len + supporters.elems.keepIf( + item => item["display_name"].getStr != "Anonymous" ) - echo("Found ", before - supportLevels.elems.len, " sponsors that cancelled or didn't pay.") - echo("Found ", supportLevels.elems.len, " active sponsors!") + echo("Discarded ", before - supporters.elems.len, " anonymous sponsors.") + echo("Found ", supporters.elems.len, " named sponsors.") - supportLevels.elems.sort( - (x, y) => cmp(y["amount"].getFNum, x["amount"].getFNum) + supporters.elems.sort( + (x, y) => cmp(y["alltime_amount"].getFNum, x["alltime_amount"].getFNum) ) -proc getSupporter(supporters: JsonNode, displayName: string): JsonNode = - for supporter in supporters: - if supporter["display_name"].getStr == displayName: - return supporter - doAssert false proc quote(text: string): string = if {' ', ','} in text: @@ -81,7 +70,7 @@ proc getLevel(amount: float): int = if amount.int <= i: result = i -proc writeCsv(sponsors: seq[Sponsor]) = +proc writeCsv(sponsors: seq[Sponsor], filename="sponsors.new.csv") = var csv = "" csv.add "logo, name, url, this_month, all_time, since, level\n" for sponsor in sponsors: @@ -91,7 +80,8 @@ proc writeCsv(sponsors: seq[Sponsor]) = $sponsor.allTime.int, sponsor.since.format("MMM d, yyyy").quote, $sponsor.amount.getLevel ] - writeFile("sponsors.new.csv", csv) + writeFile(filename, csv) + echo("Written csv file to ", filename) when isMainModule: if paramCount() == 0: @@ -105,14 +95,13 @@ when isMainModule: echo("Getting sponsors...") let supporters = waitFor bountysource.getSupporters() - - var supportLevels = waitFor bountysource.getSupportLevels() - processLevels(supportLevels) + processSupporters(supporters) echo("Generating sponsors list... (please be patient)") - var sponsors: seq[Sponsor] = @[] - for supportLevel in supportLevels: - let name = supportLevel["owner"]["display_name"].getStr + var activeSponsors: seq[Sponsor] = @[] + var inactiveSponsors: seq[Sponsor] = @[] + for supporter in supporters: + let name = supporter["display_name"].getStr var url = "" let ghUser = waitFor getGithubUser(name) if not ghUser.isNil: @@ -124,21 +113,28 @@ when isMainModule: if url.len > 0 and not url.startsWith("http"): url = "http://" & url - let amount = supportLevel["amount"].getFNum + let amount = supporter["monthly_amount"].getFNum() # Only show URL when user donated at least $5. if amount < 5: url = "" - let supporter = getSupporter(supporters, supportLevel["owner"]["display_name"].getStr) + #let supporter = getSupporter(supporters, + # supportLevel["owner"]["display_name"].getStr) + #if supporter.isNil: continue var logo = "" if amount >= 75: discard # TODO - sponsors.add(Sponsor(name: name, url: url, logo: logo, amount: amount, - allTime: supporter["alltime_amount"].getFNum(), - since: parse(supporter["created_at"].getStr, "yyyy-MM-dd'T'hh:mm:ss") - )) - - echo("Generated ", sponsors.len, " sponsors") - writeCsv(sponsors) - echo("Written csv file to sponsors.new.csv") + let sponsor = Sponsor(name: name, url: url, logo: logo, amount: amount, + allTime: supporter["alltime_amount"].getFNum(), + since: parse(supporter["created_at"].getStr, "yyyy-MM-dd'T'hh:mm:ss") + ) + if supporter["monthly_amount"].getFNum > 0.0: + activeSponsors.add(sponsor) + else: + inactiveSponsors.add(sponsor) + + echo("Generated ", activeSponsors.len, " active sponsors") + echo("Generated ", inactiveSponsors.len, " inactive sponsors") + writeCsv(activeSponsors) + writeCsv(inactiveSponsors, "inactive_sponsors.new.csv") diff --git a/web/download.rst b/web/download.rst index 6593a928c..cf0841577 100644 --- a/web/download.rst +++ b/web/download.rst @@ -3,22 +3,17 @@ Download the compiler You can download the latest version of the Nim compiler here. -**Note:** The Nim compiler requires a C compiler to compile software. On -Windows we recommend that you use -`Mingw-w64 <http://mingw-w64.sourceforge.net/>`_. GCC is recommended on Linux -and Clang on Mac. - - Binaries -------- -Unfortunately, right now we only provide binaries for Windows. You can download -an installer for both 32 bit and 64 bit versions of Windows below. +Right now binaries are only provided for Windows. You can download +an installer for both 32 bit and 64 bit versions of Windows below. These +installers have everything you need to use Nim, including a C compiler. -* | 32 bit: `nim-0.14.2_x32.exe <download/nim-0.14.2_x32.exe>`_ - | SHA-256 ca2de37759006d95db98732083a6fab20151bb9819186af2fa29d41884df78c9 -* | 64 bit: `nim-0.14.2_x64.exe <download/nim-0.14.2_x64.exe>`_ - | SHA-256 1fec054d3a5f54c0a67a40db615bb9ecb1d56413b19e324244110713bd4337d1 +* | 32 bit: `nim-0.15.0_x32.exe <download/nim-0.15.0_x32.exe>`_ + | SHA-256 0ca8931e3369735bbafdf93de98a8fd0f425870f1173845e7601922a5e00c3c2 +* | 64 bit: `nim-0.15.0_x64.exe <download/nim-0.15.0_x64.exe>`_ + | SHA-256 7bb9321cd9fb2860d36ee9b248e0202d7d4e36e2272a2f128fbce96fd4a9bfd6 These installers also include Aporia, Nimble and other useful Nim tools to get you started with Nim development! @@ -26,14 +21,18 @@ you started with Nim development! Installation based on generated C code -------------------------------------- -This installation method is the preferred way for Linux, Mac OS X, and other Unix -like systems. Binary packages may be provided later. +**Note:** The Nim compiler requires a C compiler to compile software. On +Windows we recommend that you use +`Mingw-w64 <http://mingw-w64.sourceforge.net/>`_. GCC is recommended on Linux +and Clang on Mac. The Windows installers above already includes a C compiler. +This installation method is the preferred way for Linux, Mac OS X, and other Unix +like systems. Firstly, download this archive: -* | `nim-0.14.2.tar.xz (4.5MB) <download/nim-0.14.2.tar.xz>`_ - | SHA-256 8f8d38d70ed57164795fc55e19de4c11488fcd31dbe42094e44a92a23e3f5e92 +* | `nim-0.15.0.tar.xz (4.5MB) <download/nim-0.15.0.tar.xz>`_ + | SHA-256 c514535050b2b2156147bbe6e23aafe07cd996b2afa2c81fa9a09e1cd8c669fb Extract the archive. Then copy the extracted files into your chosen installation directory, ideally somewhere in your home directory. @@ -45,6 +44,7 @@ Now open a terminal and follow these instructions: ``cd ~/programs/nim``. * run ``sh build.sh``. * Add ``$your_install_dir/bin`` to your PATH. +* To build associated tools like ``nimble`` and ``nimsuggest`` run ``nim e install_tools.nims``. After restarting your terminal, you should be able to run ``nim -v`` which should show you the version of Nim you just installed. @@ -70,9 +70,9 @@ directory where you would like the download to take place. The following commands can be used to download the current development branch and then to build it:: - git clone git://github.com/nim-lang/Nim.git + git clone https://github.com/nim-lang/Nim.git cd Nim - git clone --depth 1 git://github.com/nim-lang/csources + git clone --depth 1 https://github.com/nim-lang/csources cd csources && sh build.sh cd .. bin/nim c koch @@ -86,7 +86,7 @@ Docker Hub ---------- The `official Docker images <https://hub.docker.com/r/nimlang/nim/>`_ -are published Docker Hub and include the compiler and Nimble. There are images +are published on Docker Hub and include the compiler and Nimble. There are images for standalone scripts as well as Nimble packages. Get the latest stable image:: diff --git a/web/inactive_sponsors.csv b/web/inactive_sponsors.csv new file mode 100644 index 000000000..d466f3f31 --- /dev/null +++ b/web/inactive_sponsors.csv @@ -0,0 +1,50 @@ +logo, name, url, this_month, all_time, since, level +,bogen,,0,1010,"Jul 23, 2016",1 +,mikra,,0,400,"Apr 28, 2016",1 +,linkmonitor,,0,180,"Jan 28, 2016",1 +,avsej,,0,110,"Jun 10, 2016",1 +,WilRubin,,0,100,"Aug 11, 2015",1 +,"Benny Luypaert",,0,100,"Apr 10, 2016",1 +,"Chris Heller",,0,100,"May 19, 2016",1 +,PhilipWitte,,0,100,"Aug 5, 2016",1 +,Boxifier,,0,75,"Apr 12, 2016",1 +,iolloyd,,0,75,"Apr 29, 2016",1 +,rb01,,0,50,"May 4, 2016",1 +,TedSinger,,0,45,"Apr 9, 2016",1 +,martinbbjerregaard,,0,35,"Jun 9, 2016",1 +,RationalG,,0,30,"Jun 17, 2016",1 +,benbve,,0,30,"Jul 12, 2016",1 +,barcharcraz,,0,25,"Jun 2, 2016",1 +,"Landon Bass",,0,25,"Jun 7, 2016",1 +,jimrichards,,0,25,"Jun 8, 2016",1 +,jjzazuet,,0,25,"Jul 10, 2016",1 +,moigagoo,,0,20,"May 13, 2016",1 +,kteza1,,0,20,"Jun 10, 2016",1 +,tomkeus,,0,20,"Sep 4, 2016",1 +,mirek,,0,15,"Apr 9, 2016",1 +,DateinAsia,,0,15,"Jul 30, 2016",1 +,rickc,,0,15,"Jul 31, 2016",1 +,jpkx1984,,0,13,"Jul 11, 2016",1 +,vlkrav,,0,12,"Aug 9, 2015",1 +,tebanep,,0,12,"Aug 7, 2016",1 +,McSpiros,,0,10,"Apr 6, 2016",1 +,"Brandon Hunter",,0,10,"Apr 7, 2016",1 +,funny-falcon,,0,10,"Apr 7, 2016",1 +,teroz,,0,10,"Apr 8, 2016",1 +,iLikeLego,,0,10,"Apr 16, 2016",1 +,Angluca,,0,10,"May 3, 2016",1 +,calind,,0,10,"Jun 7, 2016",1 +,goldenreign,,0,10,"Jun 10, 2016",1 +,Blumenversand,,0,10,"Jul 21, 2016",1 +,cinnabardk,,0,10,"Aug 6, 2016",1 +,reddec,,0,10,"Aug 31, 2016",1 +,niv,,0,5,"Apr 6, 2016",1 +,goniz,,0,5,"Apr 7, 2016",1 +,genunix,,0,5,"Apr 12, 2016",1 +,CynepHy6,,0,5,"Apr 14, 2016",1 +,ivanflorentin,,0,5,"May 3, 2016",1 +,stevenyhw,,0,5,"May 20, 2016",1 +,"Sanjay Singh",,0,5,"Jun 6, 2016",1 +,yuttie,,0,5,"Jun 7, 2016",1 +,hron,,0,5,"Jun 11, 2016",1 +,laszlowaty,,0,5,"Jun 17, 2016",1 diff --git a/web/news.rst b/web/news.rst index 95822a459..f819c384c 100644 --- a/web/news.rst +++ b/web/news.rst @@ -2,6 +2,12 @@ News ==== +`2016-09-30 Nim Version 0.15.0 released <news/2016_09_00_version_0_15_0_released.html>`_ +=================================== + +`2016-09-03 Nim Community Survey results <news/2016_09_03_nim_community_survey_results.html>`_ +=================================== + `2016-08-06 BountySource Update: The Road to v1.0 <news/2016_08_06_bountysource_update_the_road_to_v10.html>`_ =================================== diff --git a/web/news/2016_09_03_nim_community_survey_results.rst b/web/news/2016_09_03_nim_community_survey_results.rst new file mode 100644 index 000000000..106dce0e4 --- /dev/null +++ b/web/news/2016_09_03_nim_community_survey_results.rst @@ -0,0 +1,699 @@ +Nim Community Survey Results +============================ + +.. container:: metadata + + Posted by Dominik Picheta on 3 September 2016 + +We have recently closed the 2016 Nim Community Survey. I am happy to +say that we have received exactly 790 responses, huge thanks go to the people +that took the time to respond. We're incredibly thankful for this very valuable +feedback. + +This survey was inspired in part by the +`2016 State of Rust <https://blog.rust-lang.org/2016/06/30/State-of-Rust-Survey-2016.html>`_ +survey. You will note that many of the questions were modelled after +Rust's survey. One of the reasons for doing this was to allow us to easily +compare our results against the results obtained in the Rust survey. In +addition, we of course also liked many of their questions. + +Our survey ran from the 23rd of June 2016 until the 8th of August 2016. The +response numbers are impressive considering Nim's community size; at 790 they +make up just over 25% of the Rust survey's responses. + +The goal of this survey was to primarily determine how our community is using +Nim, in order to better understand how we should be improving it. In particular, +we wanted to know what people feel is missing from Nim in the lead up to +version 1.0. We have also asked our respondents about how well the Nim tools +worked, the challenges of adopting Nim, the resources that they used to learn +Nim and more. + +It is my hope that we will be able to run a similar survey in a years time, +doing so should give us an idea of whether we are improving. +With these general facts in mind, let's begin looking at specific questions. + +How did you find out about Nim? +------------------------------- + +The rationale for the first question was simple, we wanted to know where our +respondents found out about Nim. This is an interesting question for us, as +we do occassionally get users asking us why it took so long for them to hear +about Nim. It allows us to see how effective each website is at spreading the +word about Nim. + +.. raw::html + + <a href="../assets/news/images/survey/nim_found.png"> + <img src="../assets/news/images/survey/nim_found.png" alt="How did you find out about Nim?" style="width:100%"/> + </a> + +The majority of our respondents found Nim via Reddit, HackerNews or a search +engine such as Google. These results are not altogether surprising. There were +also a lot of "Other" responses, some of which were a bit more +interesting. These included multiple mentions of habrahabr.ru, Dr. Dobb's, +and lobste.rs. + +Do you use Nim? +--------------- + +Just like the Rust survey creators, we wanted to ensure that our survey was +open to both Nim users as well people who never used Nim. In addition to +those two groups, we have also included a third group of people: ex-Nim +users. All three are interesting, for many different reasons. +Nim users can tell us how they are using Nim and also how Nim's +tooling can improve. Ex-Nim users give us an +idea of why they stopped using Nim. Finally, respondents who never used Nim +can tell us the reasons for not adopting it. + +.. raw::html + + <a href="../assets/news/images/survey/do_you_use_nim.png"> + <img src="../assets/news/images/survey/do_you_use_nim.png" alt="Do you use Nim?" style="width:100%"/> + </a> + +It's nice to see that we have such a good range of respondents. The Rust survey +had a much larger number of Rust users amongst their respondents, with +no distinction between users that never used Rust and users that stopped using +Rust. + +Should we consider your answers to be invalid? +---------------------------------------------- + +This was something I thought would be interesting to have, after I saw it +being used in another survey. While it does pinpoint possibly +invalid respondents, I have opted against filtering those out. Mainly because +that would require re-creating each of the charts generated by Google Forms +manually. + +.. raw::html + + <a href="../assets/news/images/survey/reliability.png"> + <img src="../assets/news/images/survey/reliability.png" alt="Should we consider your answers to be invalid?" style="width:100%"/> + </a> + +According to the responses to this question, around 94% of our responses +can be considered reliable. + +Nim users +--------- + +The following questions were answered only by the 38.9% of our respondents +who identified themselves as Nim users. + +How long have you been using Nim? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw::html + + <a href="../assets/news/images/survey/nim_time.png"> + <img src="../assets/news/images/survey/nim_time.png" alt="How long have you been using Nim?" style="width:100%"/> + </a> + +A large proportion of our Nim users were new. This is good news as it means that +our community is growing, with a large proportion of new Nim users that could +become long-term Nimians. In total, more than 35% of Nim users can be considered +new having used Nim for less than 3 months. With 18% of Nim users that can +be considered very new having used Nim for less than a month. +This could suggest that 18% of our users have only just found out about Nim in +the last week or so and have not yet got the chance to use it extensively. + +The high percentages of long term Nim users are encouraging. +They suggest +that many users are continuing to use Nim after making it through the first +few months. The sharp drop at 7-9 months is interesting, but may simply be +due to the fact that there were fewer newcomers during that period, or it +could be because our respondents are more likely to estimate that they have +been using Nim for a year or half a year rather than the awkward 7-9 months. + +.. raw::html + + <a href="../assets/news/images/survey/nim_time_rust.png"> + <img src="../assets/news/images/survey/nim_time_rust.png" alt="Time using Nim and Rust" style="width:100%"/> + </a> + +The results for Nim and Rust are actually remarkably similar. They both show a +drop at 7-9 months, although Rust's isn't as dramatic. Nim on the other hand +has a significantly higher percentage of new Nim users. + +Do you use Nim at work? +~~~~~~~~~~~~~~~~~~~~~~~ + +An important aspect of a language's adoption is whether it is being used for +"real" work. We wanted to know how many people are using Nim in their day +jobs and under what circumstances it is used. + +.. raw::html + + <a href="../assets/news/images/survey/nim_at_work.png"> + <img src="../assets/news/images/survey/nim_at_work.png" alt="Do you use Nim at work?" style="width:100%"/> + </a> + +While a vast majority of our users are not using Nim at work, more than 25% +of them are. It's encouraging to see such a high number already, even before +we have released version 1.0. In fact, this percentage is likely close to 30%, +because many of the "Other" responses mention using Nim for the likes of +internal tools or small scripts to help with the respondent's work. + +.. raw::html + + <a href="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png"> + <img src="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png" alt="Do you use Rust at work?" style="width:100%"/> + </a> + +Interestingly, a larger percentage of Nim users are using Nim at work than +Rust users. The sample sizes are of course vastly different, but it's still an +interesting result. Combined, nearly 1/5th of Rust users are using Rust +commercially whereas more than a quarter of Nim users are using Nim +commercially. + +Approximately how large are all the Nim projects that you work on? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Finding out how large the Nim projects worked on by Nim users are is also +very valuable. + +.. raw::html + + <a href="../assets/news/images/survey/project_size.png"> + <img src="../assets/news/images/survey/project_size.png" alt="Nim project size for all users" style="width:100%"/> + </a> + +This shows us that currently Nim is primarily being used for small scripts and +applications, with nearly 60% of the projects consisting of less than 1,000 +lines of code. This makes sense as many of our users are not using Nim +professionally, but are doing so in their spare time. + +.. raw::html + + <a href="../assets/news/images/survey/project_size_work.png"> + <img src="../assets/news/images/survey/project_size_work.png" alt="Nim project size for work users" style="width:100%"/> + </a> + +The numbers for part-time and full-time work users of Nim tell a different +story. Over 70% of the projects written by full-time users are between 10,001 +and 100,000 lines of code. Part-time users show a slightly different trend, +with many more small projects, the majority being between 1,000 and +10,000 lines of code. + +Overall it's good to see that there is a few large projects out there which are +composed of more than 100,000 lines of code. We expect to see the amount of +large projects to grow with time, especially with version 1.0 on the way. + +.. raw::html + + <a href="../assets/news/images/survey/project_size_nim_rust.png"> + <img src="../assets/news/images/survey/project_size_nim_rust.png" alt="Nim project size for work users (Nim vs. Rust)" style="width:100%"/> + </a> + +In comparison to Rust the proportion of project sizes for full-time users is +vastly different. This is likely due to our small sample size. Project sizes for +part-time users between Rust and Nim are somewhat similar, with differences of +around 10% for each project size. + +Do you plan to try to use Nim at work? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. raw::html + + <a href="../assets/news/images/survey/planning_to_use_at_work.png"> + <img src="../assets/news/images/survey/planning_to_use_at_work.png" alt="Planning to use Nim at work?" style="width:100%"/> + </a> + +It's also encouraging to see that over 50% of Nim users are planning to use +Nim at work! This is slightly more than Rust's 40% and should help Nim's +adoption into even more areas. + +Nim and its tools +~~~~~~~~~~~~~~~~~ + +In this section of the survey, we wanted to find out the tools that Nim +users are utilising when developing Nim applications. + +What editor(s) do you use when writing Nim? +___________________________________________ + +Programmers are very specific when it comes to their editor of choice, because +of that it's good to know which editor is most popular among our community. + +.. raw::html + + <a href="../assets/news/images/survey/editors.png"> + <img src="../assets/news/images/survey/editors.png" alt="Editors used by Nim users" style="width:100%"/> + </a> + +Looks like Vim is the winner with almost 30%. Followed by Sublime Text and +Emacs. Aporia, the Nim IDE, gets a respectable 15.5%. There was +also more than +17% of answers which included "Other" editors, such as: Notepad++, Geany, gedit, +and Kate. + +What operating system(s) do you compile for and run your Nim projects on? +_________________________________________________________________________ + +This question gave us information about the most popular target operating +systems, as well as some of the more obscure ones. We have asked this question +to find out the platforms on which Nim applications run on most frequently. + +.. raw::html + + <a href="../assets/news/images/survey/target_os.png"> + <img src="../assets/news/images/survey/target_os.png" alt="Target operating systems" style="width:100%"/> + </a> + +This question allowed multiple choices, so each percentage is out of the total +number of respondents for this question. For example, 80.7% of the +respondents selected "Linux" but only 26.6% selected OS X. + +This makes Linux by far the most popular target for Nim applications. +Some "Other" targets included: BSD (OpenBSD, FreeBSD), iOS, Android, and +JavaScript. +It's great to see Nim being used on such a wide variety of platforms. + +What operating system(s) do you develop Nim projects on? +________________________________________________________ + +With this question, we wanted to know what operating systems are used for +development. + +.. raw::html + + <a href="../assets/news/images/survey/dev_os.png"> + <img src="../assets/news/images/survey/dev_os.png" alt="Development operating systems" style="width:100%"/> + </a> + +This question also allowed multiple choices and ended up with very similar +results. + +You can see that Linux is also the most popular developmental +platform for Nim. But it's more popular as a target platform. + +Which version(s) of Nim do you use for your applications? +_________________________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/nim_versions.png"> + <img src="../assets/news/images/survey/nim_versions.png" alt="Version use" style="width:100%"/> + </a> + +At the time of this survey, version 0.14.2 was the latest stable release. +It's no wonder that it is the most commonly used release of Nim. It's good to +see that the older versions are not used as often. The high use of ``Git HEAD (devel)`` +(nightly builds) isn't surprising, Nim is still evolving rapidly and our +release schedule is not regular or frequent. + +Once we go past the 1.0 release, we expect to see much less use of the unstable +``devel`` branch. + +Has upgrading to a new version of the Nim compiler broken your code? +____________________________________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/breakage.png"> + <img src="../assets/news/images/survey/breakage.png" alt="Breakage" style="width:100%"/> + </a> + +Despite the unstable nature of Nim in the lead up to version 1.0, whenever +we make breaking changes we do our best to deprecate things and ensure that +old code continues to work for our users. Of course sometimes this is not +possible and other times it is simply easier to add a breaking change. + +This question was asked to determine how much our user base is affected by +breaking changes between Nim versions. We decided to have three possible +answers for this question in order to give us an idea how frequent the +breakage was. + +It's incredible to see that over 50% of our users have not experienced any +breakage after upgrading. We expect this number to increase significantly +after version 1.0 is released. Of the users that did experience breakage, +over 80% of them said that it was a rare occurrence. + +In comparison to Rust, our results show that there was a higher percentage of +users experiencing breakage as a result of an upgrade. This is to be expected, +because Nim is still in its pre-1.0 period, whereas Rust 1.0 has been released +over a year ago now. + +Unfortunately while we are still in this pre-1.0 period, releases will likely +introduce breaking changes as we refine certain aspects of Nim such as its +standard library, so the number of users experiencing breaking changes may +increase. + +If so, how much work did it take to fix it? +___________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/difficulty_fixing_breakage.png"> + <img src="../assets/news/images/survey/difficulty_fixing_breakage.png" alt="difficulty fixing breakage" style="width:100%"/> + </a> + +Thankfully most of the breakage experienced by Nim users was very easy to fix. + + +If you used Nimble, do you like it? +___________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/nimble_opinion.png"> + <img src="../assets/news/images/survey/nimble_opinion.png" alt="Do you like Nimble?" style="width:100%"/> + </a> + +Nimble is the Nim package manager, a tool that is very important in Nim's +ecosystem as it allows developers to easily install dependencies for their +software. + +The majority of respondents rated it as a 4, showing us that the majority does +like Nimble. With over 55% rating it a 4 or 5. This percentage isn't as +overwhelming as the 94.1% of users that rated Cargo a 4 or 5 in the Rust +survey. Based on these results I think that we definitely need to do a +better job with Nimble. + +In our next survey, it might be a good idea to ask more questions about Nimble +to determine how exactly it can be improved. + +What aspects of Nim do you find most appealing? +_______________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/nim_appeal.png"> + <img src="../assets/news/images/survey/nim_appeal.png" alt="What aspects of Nim do you find most appealing?" style="width:100%"/> + </a> + +We were interested to know the features of Nim that appeal most to our users. +More than 80% of our respondents selected "Execution Speed" as one of the +features that appeal to them. With "Development Speed" and "Readability" +tying for second place and "Metaprogramming" at third place. + +The options given to our respondents are rather predictable, +they do show us which of these features have the highest appeal though. +What's more interesting are the "Other" answers. + +By far the most popular "Other" answer was related to Nim's compilation to C. +Many users mentioned that they like how easy it is to interface with C +libraries and the great portability that compiling to C offers. + +What aspects of Nim do you find most displeasing? +_________________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/nim_displeasing.png"> + <img src="../assets/news/images/survey/nim_displeasing.png" alt="What aspects of Nim do you find most displeasing?" style="width:100%"/> + </a> + +It was only natural to ask this question. The results are almost perfectly +opposite to the previous question's answers, with almost 50% of respondents +selecting "Debugging Tools" +as the most displeasing aspect of Nim. With "Documentation" and "Testing Tools" +in second and third place respectively. There is also a much larger number of +"Other" answers to this question. + +The "Other" answers for this question vary a lot. Here is a selection of +them, ordered by frequency: + +* Small community size. +* Lack of in-depth tutorials. +* Quality of error messages. +* Forward declarations and no cyclic imports. +* Bugs in the standard library. +* No good IDE. +* No REPL. +* No major version. +* Bugs in the compiler. +* Lack of libraries. +* Difficulty installing on Windows. +* Non-intuitive semantics of various constructs. +* Lack of immutable collections. +* Async/await not being production ready. +* Lack of shared collections for threads. +* No Haxe target. +* Memory safety. + +We hope that we can improve these things with time. Many of these issues are +already being worked on, including the removal of the need for forward +declarations. Some of these issues like our small community size are difficult +to fix, but we will nonetheless do our best. + + +Previous Nim users +~~~~~~~~~~~~~~~~~~ + +For users that have used Nim before but decided against using it, we asked just +one specific question. The proportion of our respondents that answered it +was 24%. + +Why did you stop using Nim? +___________________________ + +.. raw::html + + <a href="../assets/news/images/survey/ex_nim.png"> + <img src="../assets/news/images/survey/ex_nim.png" alt="I stopped using Nim because..." style="width:100%"/> + </a> + +Again, this question got a lot of "Other" answers. Apart from that, the +most popular reason for leaving Nim is that it is not stable. Followed by the +a lack of needed libraries and packages and the instability of the +standard library. + +* Lack of IDE support. +* Style insensitive. +* Documentation. +* Dislike the syntax. +* Community is too small. +* Missing language features (for example RAII). +* No opportunities to use it at work. +* Messy standard library. + +The first item, "Lack of IDE support", was mentioned by multiple respondents. +In the future we should look into ensuring that major IDEs have plugins which +enable easy Nim development. + +Based on some of the "Other" answers, it seems that many of the respondents +have not used Nim for very long, for example many respondents complained about +installation issues which they would have run into before getting a chance to +use Nim. Because of this I would consider them not +ex-Nim users but developers that have not had a chance to try Nim fully. +Next time we should also ask how long the respondent has used Nim for to get a +better idea of whether they had a chance to use Nim for extended periods of +time. + +Non-Nim users +~~~~~~~~~~~~~ + +We also wanted to know the reasons why developers decided against using Nim. + +Why do you not use Nim? +_______________________ + +.. raw::html + + <a href="../assets/news/images/survey/non_user.png"> + <img src="../assets/news/images/survey/non_user.png" alt="I don't use Nim because..." style="width:100%"/> + </a> + +The most common reason that people have for not using Nim is that it is +not yet ready for production. Thankfully this will improve with time. +IDE support is also a prominent factor just as we've seen in previous results. + +There is also a lot of "Other" answers, let's have a look at a selection of +them. Some of the most prominent ones, in order of frequency, include: + +* No time to use/learn it +* Syntax +* Documentation is incomplete +* Garbage Collection +* Prefer functional paradigm +* Small community +* Style insensitivity/Case insensitivity + +One respondent made a very good suggestion: they said that the +"Do you use Nim?" question should have included "No, but I intend to" as +an answer. Definitely something we will do in the next survey. Indeed, many +respondents mentioned that they were planning on trying out Nim but that they +just have no time to do so, this is very encouraging! + +Learning Resources +~~~~~~~~~~~~~~~~~~ + +We wanted to get an idea of how Nim users are learning Nim. Every respondent +answered this question, no matter what they answered for the "Do you use Nim?" +question. + +Which learning resources, if any, did you use to learn Nim? +___________________________________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/learning_resources.png"> + <img src="../assets/news/images/survey/learning_resources.png" alt="learning resources" style="width:100%"/> + </a> + +The idea behind this question was to understand which learning resources +were most popular among our user base. The +`Nim tutorial <http://nim-lang.org/docs/tut1.html>`_ is by far the most +popular. In previous questions, we saw respondents mentioning that the Nim +tutorial does not go into enough detail about Nim. Thanks to this information +we can come to the conclusion that the tutorial needs to be improved +significantly to make sure that it gives our users the necessary information +to use Nim effectively. + +Indeed, many users also use the +`Nim manual <http://nim-lang.org/docs/manual.html>`_ to learn Nim. +This manual has been +written as a specification and so is not ideal for teaching Nim. Many of +the concepts in the Nim manual need to be explained in a lot more detail in +the Nim tutorial. + +Of course, it's exciting to see our respondents using other materials to learn +Nim. In particular I am excited to see that over 15% of the respondents have +used +`Nim in Action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_ +to learn Nim. I expect that more and more users will pick up the book after it +is fully published. + +Nim in Action +_____________ + +As the author of +`Nim in Action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_, +I wanted to get some statistics surrounding +my book. With this in mind, I have created some questions relating to it. + +Have you read Nim in Action? +____________________________ + +.. raw::html + + <a href="../assets/news/images/survey/book.png"> + <img src="../assets/news/images/survey/book.png" alt="Have you read Nim in Action?" style="width:100%"/> + </a> + +It's good to see that over 50% of respondents have read the book or are at least +planning to read it. Keep in mind that this question was answered by all +respondents, not just Nim users. + +.. container:: standout + + Are you interested in purchasing a copy of + `Nim in Action <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_? + If so, you can use code ``wm090416lt`` to get 50% off the printed book today only! + If you purchase it now you will get access to an early access copy of + Nim in Action in eBook form and will be able to take part in the development + of this book. + +Did you enjoy Nim in Action? +____________________________ + +.. raw::html + + <a href="../assets/news/images/survey/book_opinion.png"> + <img src="../assets/news/images/survey/book_opinion.png" alt="Did you enjoy Nim in Action?" style="width:100%"/> + </a> + +Of the people that read Nim in Action it's nice to see that almost 70% have +enjoyed it. + +Nim's future +~~~~~~~~~~~~ + +What improvements are needed before Nim v1.0 can be released? +_____________________________________________________________ + +We were interested to know what our users believe is needed before +Nim version 1.0 can be released. + +.. raw::html + + <a href="../assets/news/images/survey/10_needs.png"> + <img src="../assets/news/images/survey/10_needs.png" alt="What is needed before 1.0 can be released?" style="width:100%"/> + </a> + +It appears that the standard library is the biggest concern. With more than half +of all respondents selecting "The standard library needs to reviewed and +any problems with it fixed". This is in fact something we are already planning +to address, so it's good to see that the majority agrees with us. + +A large proportion of users also believes that the language is great as-is +and that we should focus on stabilising the compiler. This somewhat contradicts +the majority. But perhaps most of them thought that "The language" excludes the +standard library. + +For this question, we decided to give our respondents a dedicated place to +give general feedback about what they feel is needed before v1.0 can be +released. We received over 200 responses to that. Many of these responses +reflect what we have already seen: that the documentation needs to improve, +that we need a good Nim IDE, stability for experimental features such as +concepts, the standard library needs to be cleaned up. + +Unfortunately many respondents used this question to say what needs to be fixed +in Nim in general, not what is definitely necessary before 1.0 can be released. + +Community demographics +~~~~~~~~~~~~~~~~~~~~~~ + +What domain do you work in currently? +_____________________________________ + +.. raw::html + + <a href="../assets/news/images/survey/domains.png"> + <img src="../assets/news/images/survey/domains.png" alt="Work domains" style="width:100%"/> + </a> + + +Nim users are working in a wide variety of domains. It is encouraging to see +people from so many different backgrounds taking part in this survey. + +What programming languages are you most comfortable with? +_________________________________________________________ + + +.. raw::html + + <a href="../assets/news/images/survey/languages.png"> + <img src="../assets/news/images/survey/languages.png" alt="Programming languages" style="width:100%"/> + </a> + +Python and C are the top two programming languages that our respondents are +most comfortable with. This is not altogether surprising. + +Last words +~~~~~~~~~~ + +At the end of the survey we gave our respondents a chance to speak their mind +about anything they wish, with a simple question: "Anything else you'd like +to tell us?" + +There was a lot of great feedback given in this question from people who +obviously really care deeply about Nim. There is too much to outline here, +but rest assurred that we will take it all into account and do our best to +act on it. + +In addition to feedback, we were also overwhelmed by the amount of positive +comments in the answers to this +question. There was a lot of support from the community thanking us for our +work and determination. + +I'll let some quotes speak for themselves: + +.. raw::html + + <blockquote>You rock, seriously.</blockquote> + <blockquote>Nim rocks! Keep it up! Thank you very much!</blockquote> + <blockquote>You've made great progress on the language without any corporate backing, that is amazing. I wish Nim becomes one of the top used languages in a few years.</blockquote> + <blockquote>Nim is elegant and wonderful! Keep at it!</blockquote> + +Our community is truly brilliant. We thank each and every one of you for +filling out this survey and hope that you will help us tackle some of the +challenges that face Nim. + +This survey was a good place to give us feedback, but please don't wait for +the next one. We are always looking to hear more from you and we hope that you +will participate in discussions relating to this survey as well the future +of Nim. + +Thanks for reading, and have a good day! diff --git a/web/news/2016_09_30_version_0_15_0_released.rst b/web/news/2016_09_30_version_0_15_0_released.rst new file mode 100644 index 000000000..47c02e9e4 --- /dev/null +++ b/web/news/2016_09_30_version_0_15_0_released.rst @@ -0,0 +1,517 @@ +Version 0.15.0 released +======================= + +.. container:: metadata + + Posted by Dominik Picheta and Andreas Rumpf on 30/09/2016 + +We're happy to announce that the latest release of Nim, version 0.15.0, is now +available! + +As always, you can grab the latest version from the +`downloads page <http://nim-lang.org/download.html>`_. + +This release includes almost 180 bug fixes and improvements. To see a full list +of changes, take a look at the detailed changelog +`below <#changelog>`_. + +Some of the most significant changes in this release include: improvements to +the documentation, addition of a new ``multisync`` macro, and a new +``HttpClient`` implementation. + +Documentation +~~~~~~~~~~~~~ + +All pages in the documentation now contain a search box and a drop down to +select how procedures should be sorted. This allows you to search for +procedures, types, macros and more from any documentation page. + +.. raw::html + + <a href="../assets/news/images/0.15.0/doc_search.gif"> + <img src="../assets/news/images/0.15.0/doc_search.gif" alt="Doc search" style="width:100%"/> + </a> + +Sorting the procedures by type shows a more natural table of contents. This +should also help you to find procedures and other identifiers. + +.. raw::html + + <a href="../assets/news/images/0.15.0/doc_sort.gif"> + <img src="../assets/news/images/0.15.0/doc_sort.gif" alt="Doc sort" style="width:100%"/> + </a> + +Multisync macro +~~~~~~~~~~~~~~~ + +The ``multisync`` macro was implemented to enable you to define both +synchronous and asynchronous IO procedures without having to duplicate a +lot of code. + +As an example, consider the ``recvTwice`` procedure below: + +.. code-block:: nim + proc recvTwice(socket: Socket | AsyncSocket): Future[string] {.multisync.} = + result = "" + result.add(await socket.recv(25)) + result.add(await socket.recv(20)) + +The ``multisync`` macro will transform this procedure into the following: + +.. code-block:: nim + proc recvTwice(socket: Socket): string = + result = "" + result.add(socket.recv(25)) + result.add(socket.recv(20)) + + proc recvTwice(socket: AsyncSocket): Future[string] {.async.} = + result = "" + result.add(await socket.recv(25)) + result.add(await socket.recv(20)) + +Allowing you to use ``recvTwice`` with both synchronous and asynchronous sockets. + +HttpClient +~~~~~~~~~~ + +Many of the ``httpclient`` module's procedures have been deprecated in +favour of a new implementation using the ``multisync`` macro. There are now +two types: ``HttpClient`` and ``AsyncHttpClient``. Both of these implement the +same procedures and functionality, the only difference is timeout support and +whether they are blocking or not. + +See the `httpclient <http://nim-lang.org/docs/httpclient.html>`_ module +documentation for more information. + +Changelog +~~~~~~~~~ + +Changes affecting backwards compatibility +----------------------------------------- + +- The ``json`` module now uses an ``OrderedTable`` rather than a ``Table`` + for JSON objects. + +- The ``split`` `(doc) <http://nim-lang.org/docs/strutils.html#split,string,set[char],int>`_ + procedure in the ``strutils`` module (with a delimiter of type + ``set[char]``) no longer strips and splits characters out of the target string + by the entire set of characters. Instead, it now behaves in a + similar fashion to ``split`` with ``string`` and ``char`` + delimiters. Use ``splitWhitespace`` to get the old behaviour. + +- The command invocation syntax will soon apply to open brackets + and curlies too. This means that code like ``a [i]`` will be + interpreted as ``a([i])`` and not as ``a[i]`` anymore. Likewise + ``f (a, b)`` means that the tuple ``(a, b)`` is passed to ``f``. + The compiler produces a warning for ``a [i]``:: + + Warning: a [b] will be parsed as command syntax; spacing is deprecated + + See `Issue #3898 <https://github.com/nim-lang/Nim/issues/3898>`_ for the + relevant discussion. + +- Overloading the special operators ``.``, ``.()``, ``.=``, ``()`` now + needs to be enabled via the ``{.experimental.}`` pragma. + +- ``immediate`` templates and macros are now deprecated. + Use ``untyped`` `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ + parameters instead. + +- The metatype ``expr`` is deprecated. Use ``untyped`` + `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ instead. + +- The metatype ``stmt`` is deprecated. Use ``typed`` + `(doc) <http://nim-lang.org/docs/manual.html#templates-typed-vs-untyped-parameters>`_ instead. + +- The compiler is now more picky when it comes to ``tuple`` types. The + following code used to compile, now it's rejected: + +.. code-block:: nim + + import tables + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64]]() + rocketaims["hi"] = {(-1.int8, 0.int8): 0.int64}.toTable() + +Instead be consistent in your tuple usage and use tuple names for named tuples: + +.. code-block:: nim + + import tables + var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64]]() + rocketaims["hi"] = {(k: -1.int8, v: 0.int8): 0.int64}.toTable() + +- Now when you compile console applications for Windows, console output + encoding is automatically set to UTF-8. + +- Unhandled exceptions in JavaScript are now thrown regardless of whether + ``noUnhandledHandler`` is defined. But the stack traces should be much more + readable now. + +- In JavaScript, the ``system.alert`` procedure has been deprecated. + Use ``dom.alert`` instead. + +- De-deprecated ``re.nim`` because there is too much code using it + and it got the basic API right. + +- The type of ``headers`` field in the ``AsyncHttpClient`` type + `(doc) <http://nim-lang.org/docs/httpclient.html#AsyncHttpClient>`_ + has been changed + from a string table to the specialised ``HttpHeaders`` type. + +- The ``httpclient.request`` + `(doc) <http://nim-lang.org/docs/httpclient.html#request,AsyncHttpClient,string,string,string>`_ + procedure which takes the ``httpMethod`` as a string + value no longer requires it to be prefixed with ``"http"`` + (or similar). + +- Converting a ``HttpMethod`` + `(doc) <nim-lang.org/docs/httpcore.html#HttpMethod>`_ + value to a string using the ``$`` operator will + give string values without the ``"Http"`` prefix now. + +- The ``Request`` + `(doc) <http://nim-lang.org/docs/asynchttpserver.html#Request>`_ + object defined in the ``asynchttpserver`` module now uses + the ``HttpMethod`` type for the request method. + +Library Additions +----------------- + +- Added ``readHeaderRow`` and ``rowEntry`` to the ``parsecsv`` + `(doc) <http://nim-lang.org/docs/parsecsv.html>`_ module + to provide + a lightweight alternative to python's ``csv.DictReader``. + +- Added ``setStdIoUnbuffered`` proc to the ``system`` module to enable + unbuffered I/O. + +- Added ``center`` and ``rsplit`` to the ``strutils`` + `(doc) <http://nim-lang.org/docs/strutils.html>`_ module + to provide similar Python functionality for Nim's strings. + +- Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, + ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` + to the ``unicode.nim`` + `(doc) <http://nim-lang.org/docs/unicode.html>`_ module + to provide unicode aware case manipulation and case + testing. + +- Added a new module ``strmisc`` + `(doc) <http://nim-lang.org/docs/strmisc.html>`_ + to hold uncommon string + operations. Currently contains ``partition``, ``rpartition`` + and ``expandTabs``. + +- Split out ``walkFiles`` in the ``os`` + `(doc) <http://nim-lang.org/docs/os.html>`_ module to three separate procs + in order to make a clear distinction of functionality. ``walkPattern`` iterates + over both files and directories, while ``walkFiles`` now only iterates + over files and ``walkDirs`` only iterates over directories. + +- Added a synchronous ``HttpClient`` in the ``httpclient`` + `(doc) <http://nim-lang.org/docs/httpclient.html>`_ + module. The old + ``get``, ``post`` and similar procedures are now deprecated in favour of it. + +- Added a new macro called ``multisync`` allowing you to write procedures for + synchronous and asynchronous sockets with no duplication. + +- The ``async`` macro will now complete ``FutureVar[T]`` parameters + automatically unless they have been completed already. + +Tool Additions +-------------- + +- The documentation is now searchable and sortable by type. +- Pragmas are now hidden by default in the documentation to reduce noise. +- Edit links are now present in the documentation. + + +Compiler Additions +------------------ + +- The ``-d/--define`` flag can now optionally take a value to be used + by code at compile time. + `(doc) <http://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ + +Nimscript Additions +------------------- + +- It's possible to enable and disable specific hints and warnings in + Nimscript via the ``warning`` and ``hint`` procedures. + +- Nimscript exports a proc named ``patchFile`` which can be used to + patch modules or include files for different Nimble packages, including + the ``stdlib`` package. + +Language Additions +------------------ + +- Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of + (optional) compile time defines. + `(doc) <http://nim-lang.org/docs/manual.html#implementation-specific-pragmas-compile-time-define-pragmas>`_ + +- If the first statement is an ``import system`` statement then ``system`` + is not imported implicitly anymore. This allows for code like + ``import system except echo`` or ``from system import nil``. + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-06-22+..+2016-09-30%22+>`_. + +- Fixed "RFC: should startsWith and endsWith work with characters?" + (`#4252 <https://github.com/nim-lang/Nim/issues/4252>`_) + +- Fixed "Feature request: unbuffered I/O" + (`#2146 <https://github.com/nim-lang/Nim/issues/2146>`_) +- Fixed "clear() not implemented for CountTableRef" + (`#4325 <https://github.com/nim-lang/Nim/issues/4325>`_) +- Fixed "Cannot close file opened async" + (`#4334 <https://github.com/nim-lang/Nim/issues/4334>`_) +- Fixed "Feature Request: IDNA support" + (`#3045 <https://github.com/nim-lang/Nim/issues/3045>`_) +- Fixed "Async: wrong behavior of boolean operations on futures" + (`#4333 <https://github.com/nim-lang/Nim/issues/4333>`_) +- Fixed "os.walkFiles yields directories" + (`#4280 <https://github.com/nim-lang/Nim/issues/4280>`_) +- Fixed "Fix #4392 and progress on #4170" + (`#4393 <https://github.com/nim-lang/Nim/issues/4393>`_) +- Fixed "Await unable to wait futures from objects fields" + (`#4390 <https://github.com/nim-lang/Nim/issues/4390>`_) +- Fixed "TMP variable name generation should be more stable" + (`#4364 <https://github.com/nim-lang/Nim/issues/4364>`_) +- Fixed "nativesockets doesn't compile for Android 4.x (API v19 or older) because of gethostbyaddr" + (`#4376 <https://github.com/nim-lang/Nim/issues/4376>`_) +- Fixed "no generic parameters allowed for ref" + (`#4395 <https://github.com/nim-lang/Nim/issues/4395>`_) +- Fixed "split proc in strutils inconsistent for set[char]" + (`#4305 <https://github.com/nim-lang/Nim/issues/4305>`_) +- Fixed "Problem with sets in devel" + (`#4412 <https://github.com/nim-lang/Nim/issues/4412>`_) +- Fixed "Compiler crash when using seq[PNimrodNode] in macros" + (`#537 <https://github.com/nim-lang/Nim/issues/537>`_) +- Fixed "ospaths should be marked for nimscript use only" + (`#4249 <https://github.com/nim-lang/Nim/issues/4249>`_) +- Fixed "Repeated deepCopy() on a recursive data structure eventually crashes" + (`#4340 <https://github.com/nim-lang/Nim/issues/4340>`_) +- Fixed "Analyzing destructor" + (`#4371 <https://github.com/nim-lang/Nim/issues/4371>`_) +- Fixed "getType does not work anymore on a typedesc" + (`#4462 <https://github.com/nim-lang/Nim/issues/4462>`_) +- Fixed "Error in rendering empty JSON array" + (`#4399 <https://github.com/nim-lang/Nim/issues/4399>`_) +- Fixed "Segmentation fault when using async pragma on generic procs" + (`#2377 <https://github.com/nim-lang/Nim/issues/2377>`_) +- Fixed "Forwarding does not work for generics, | produces an implicit generic" + (`#3055 <https://github.com/nim-lang/Nim/issues/3055>`_) +- Fixed "Inside a macro, the length of the `seq` data inside a `queue` does not increase and crashes" + (`#4422 <https://github.com/nim-lang/Nim/issues/4422>`_) +- Fixed "compiler sigsegv while processing varargs" + (`#4475 <https://github.com/nim-lang/Nim/issues/4475>`_) +- Fixed "JS codegen - strings are assigned by reference" + (`#4471 <https://github.com/nim-lang/Nim/issues/4471>`_) +- Fixed "when statement doesn't verify syntax" + (`#4301 <https://github.com/nim-lang/Nim/issues/4301>`_) +- Fixed ".this pragma doesn't work with .async procs" + (`#4358 <https://github.com/nim-lang/Nim/issues/4358>`_) +- Fixed "type foo = range(...) crashes compiler" + (`#4429 <https://github.com/nim-lang/Nim/issues/4429>`_) +- Fixed "Compiler crash" + (`#2730 <https://github.com/nim-lang/Nim/issues/2730>`_) +- Fixed "Crash in compiler with static[int]" + (`#3706 <https://github.com/nim-lang/Nim/issues/3706>`_) +- Fixed "Bad error message "could not resolve"" + (`#3548 <https://github.com/nim-lang/Nim/issues/3548>`_) +- Fixed "Roof operator on string in template crashes compiler (Error: unhandled exception: sons is not accessible [FieldError])" + (`#3545 <https://github.com/nim-lang/Nim/issues/3545>`_) +- Fixed "SIGSEGV during compilation with parallel block" + (`#2758 <https://github.com/nim-lang/Nim/issues/2758>`_) +- Fixed "Codegen error with template and implicit dereference" + (`#4478 <https://github.com/nim-lang/Nim/issues/4478>`_) +- Fixed "@ in importcpp should work with no-argument functions" + (`#4496 <https://github.com/nim-lang/Nim/issues/4496>`_) +- Fixed "Regression: findExe raises" + (`#4497 <https://github.com/nim-lang/Nim/issues/4497>`_) +- Fixed "Linking error - repeated symbols when splitting into modules" + (`#4485 <https://github.com/nim-lang/Nim/issues/4485>`_) +- Fixed "Error: method is not a base" + (`#4428 <https://github.com/nim-lang/Nim/issues/4428>`_) +- Fixed "Casting from function returning a tuple fails" + (`#4345 <https://github.com/nim-lang/Nim/issues/4345>`_) +- Fixed "clang error with default nil parameter" + (`#4328 <https://github.com/nim-lang/Nim/issues/4328>`_) +- Fixed "internal compiler error: openArrayLoc" + (`#888 <https://github.com/nim-lang/Nim/issues/888>`_) +- Fixed "Can't forward declare async procs" + (`#1970 <https://github.com/nim-lang/Nim/issues/1970>`_) +- Fixed "unittest.check and sequtils.allIt do not work together" + (`#4494 <https://github.com/nim-lang/Nim/issues/4494>`_) +- Fixed "httpclient package can't make SSL requests over an HTTP proxy" + (`#4520 <https://github.com/nim-lang/Nim/issues/4520>`_) +- Fixed "False positive warning "declared but not used" for enums." + (`#4510 <https://github.com/nim-lang/Nim/issues/4510>`_) +- Fixed "Explicit conversions not using converters" + (`#4432 <https://github.com/nim-lang/Nim/issues/4432>`_) + +- Fixed "Unclear error message when importing" + (`#4541 <https://github.com/nim-lang/Nim/issues/4541>`_) +- Fixed "Change console encoding to UTF-8 by default" + (`#4417 <https://github.com/nim-lang/Nim/issues/4417>`_) + +- Fixed "Typedesc ~= Generic notation does not work anymore!" + (`#4534 <https://github.com/nim-lang/Nim/issues/4534>`_) +- Fixed "unittest broken?" + (`#4555 <https://github.com/nim-lang/Nim/issues/4555>`_) +- Fixed "Operator "or" in converter types seems to crash the compiler." + (`#4537 <https://github.com/nim-lang/Nim/issues/4537>`_) +- Fixed "nimscript failed to compile/run -- Error: cannot 'importc' variable at compile time" + (`#4561 <https://github.com/nim-lang/Nim/issues/4561>`_) +- Fixed "Regression: identifier expected, but found ..." + (`#4564 <https://github.com/nim-lang/Nim/issues/4564>`_) +- Fixed "varargs with transformation that takes var argument creates invalid c code" + (`#4545 <https://github.com/nim-lang/Nim/issues/4545>`_) +- Fixed "Type mismatch when using empty tuple as generic parameter" + (`#4550 <https://github.com/nim-lang/Nim/issues/4550>`_) +- Fixed "strscans" + (`#4562 <https://github.com/nim-lang/Nim/issues/4562>`_) +- Fixed "getTypeImpl crashes (SIGSEGV) on variant types" + (`#4526 <https://github.com/nim-lang/Nim/issues/4526>`_) +- Fixed "Wrong result of sort in VM" + (`#4065 <https://github.com/nim-lang/Nim/issues/4065>`_) +- Fixed "I can't call the random[T](x: Slice[T]): T" + (`#4353 <https://github.com/nim-lang/Nim/issues/4353>`_) +- Fixed "invalid C code generated (function + block + empty tuple)" + (`#4505 <https://github.com/nim-lang/Nim/issues/4505>`_) + +- Fixed "performance issue: const Table make a copy at runtime lookup." + (`#4354 <https://github.com/nim-lang/Nim/issues/4354>`_) +- Fixed "Compiler issue: libraries without absolute paths cannot be found correctly" + (`#4568 <https://github.com/nim-lang/Nim/issues/4568>`_) +- Fixed "Cannot use math.`^` with non-int types." + (`#4574 <https://github.com/nim-lang/Nim/issues/4574>`_) +- Fixed "C codegen fails when constructing an array using an object constructor." + (`#4582 <https://github.com/nim-lang/Nim/issues/4582>`_) +- Fixed "Visual Studio 10 unresolved external symbol _trunc(should we support VS2010?)" + (`#4532 <https://github.com/nim-lang/Nim/issues/4532>`_) +- Fixed "Cannot pass generic subtypes to proc for generic supertype" + (`#4528 <https://github.com/nim-lang/Nim/issues/4528>`_) +- Fixed "Lamda-lifting bug leading to crash." + (`#4551 <https://github.com/nim-lang/Nim/issues/4551>`_) +- Fixed "First-class iterators declared as inline are compiled at Nim side (no error message) and fail at C" + (`#2094 <https://github.com/nim-lang/Nim/issues/2094>`_) +- Fixed "VS2010-warning C4090 : 'function' : different 'const' qualifiers" + (`#4590 <https://github.com/nim-lang/Nim/issues/4590>`_) +- Fixed "Regression: type mismatch with generics" + (`#4589 <https://github.com/nim-lang/Nim/issues/4589>`_) +- Fixed "„can raise an unlisted exception“ when assigning nil as default value" + (`#4593 <https://github.com/nim-lang/Nim/issues/4593>`_) +- Fixed "upcoming asyncdispatch.closeSocket is not GC-safe" + (`#4606 <https://github.com/nim-lang/Nim/issues/4606>`_) +- Fixed "Visual Studio 10.0 compiler errors, 12.0 warning" + (`#4459 <https://github.com/nim-lang/Nim/issues/4459>`_) +- Fixed "Exception of net.newContext: result.extraInternalIndex == 0 [AssertionError]" + (`#4406 <https://github.com/nim-lang/Nim/issues/4406>`_) +- Fixed "error: redeclaration of 'result_115076' with no linkage" + (`#3221 <https://github.com/nim-lang/Nim/issues/3221>`_) +- Fixed "Compiler crashes on conversion from int to float at compile time" + (`#4619 <https://github.com/nim-lang/Nim/issues/4619>`_) +- Fixed "wrong number of arguments regression in devel" + (`#4600 <https://github.com/nim-lang/Nim/issues/4600>`_) +- Fixed "importc $ has broken error message (and is not documented)" + (`#4579 <https://github.com/nim-lang/Nim/issues/4579>`_) +- Fixed "Compiler segfaults on simple importcpp in js mode [regression]" + (`#4632 <https://github.com/nim-lang/Nim/issues/4632>`_) +- Fixed "Critical reference counting codegen problem" + (`#4653 <https://github.com/nim-lang/Nim/issues/4653>`_) +- Fixed "tables.nim needs lots of {.noSideEffect.}" + (`#4254 <https://github.com/nim-lang/Nim/issues/4254>`_) +- Fixed "Capture variable error when using ``=>`` macro" + (`#4658 <https://github.com/nim-lang/Nim/issues/4658>`_) +- Fixed "Enum from char: internal error getInt" + (`#3606 <https://github.com/nim-lang/Nim/issues/3606>`_) +- Fixed "Compiler crashes in debug mode (no error in release mode) with Natural discriminant in object variants" + (`#2865 <https://github.com/nim-lang/Nim/issues/2865>`_) +- Fixed "SIGSEGV when access field in const object variants" + (`#4253 <https://github.com/nim-lang/Nim/issues/4253>`_) +- Fixed "varargs cannot be used with template converter." + (`#4292 <https://github.com/nim-lang/Nim/issues/4292>`_) +- Fixed "Compiler crashes when borrowing $" + (`#3928 <https://github.com/nim-lang/Nim/issues/3928>`_) +- Fixed "internal error: genMagicExpr: mArrPut" + (`#4491 <https://github.com/nim-lang/Nim/issues/4491>`_) +- Fixed "Unhelpful error message on importc namespace collision" + (`#4580 <https://github.com/nim-lang/Nim/issues/4580>`_) +- Fixed "Problem with openarrays and slices" + (`#4179 <https://github.com/nim-lang/Nim/issues/4179>`_) +- Fixed "Removing lines from end of file then rebuilding does not rebuild [js only?]" + (`#4656 <https://github.com/nim-lang/Nim/issues/4656>`_) +- Fixed "getCurrentException and getCurrentExceptionMsg do not work with JS" + (`#4635 <https://github.com/nim-lang/Nim/issues/4635>`_) +- Fixed "generic proc parameter is not inferred if type parameter has specifier" + (`#4672 <https://github.com/nim-lang/Nim/issues/4672>`_) +- Fixed "Cannot instantiate generic parameter when it is parent type parameter" + (`#4673 <https://github.com/nim-lang/Nim/issues/4673>`_) +- Fixed "deepCopy doesn't work with inheritance after last commit" + (`#4693 <https://github.com/nim-lang/Nim/issues/4693>`_) +- Fixed "Multi-methods don't work when passing ref to a different thread" + (`#4689 <https://github.com/nim-lang/Nim/issues/4689>`_) +- Fixed "Infinite loop in effect analysis on generics" + (`#4677 <https://github.com/nim-lang/Nim/issues/4677>`_) +- Fixed "SIGSEGV when compiling NimYAML tests" + (`#4699 <https://github.com/nim-lang/Nim/issues/4699>`_) + +- Fixed "Closing AsyncEvent now also unregisters it on non-Windows platforms" + (`#4694 <https://github.com/nim-lang/Nim/issues/4694>`_) +- Fixed "Don't update handle in upcoming/asyncdispatch poll() if it was closed" + (`#4697 <https://github.com/nim-lang/Nim/issues/4697>`_) +- Fixed "generated local variables declared outside block" + (`#4721 <https://github.com/nim-lang/Nim/issues/4721>`_) +- Fixed "Footer Documentation links, & Community link point to the wrong place under news entries" + (`#4529 <https://github.com/nim-lang/Nim/issues/4529>`_) +- Fixed "Jester's macro magic leads to incorrect C generation" + (`#4088 <https://github.com/nim-lang/Nim/issues/4088>`_) +- Fixed "cas bug in atomics.nim" + (`#3279 <https://github.com/nim-lang/Nim/issues/3279>`_) +- Fixed "nimgrep PEG not capturing the pattern 'A'" + (`#4751 <https://github.com/nim-lang/Nim/issues/4751>`_) +- Fixed "GC assert triggers when assigning TableRef threadvar" + (`#4640 <https://github.com/nim-lang/Nim/issues/4640>`_) +- Fixed ".this pragma conflicts with experimental ptr dereferencing when names conflict" + (`#4671 <https://github.com/nim-lang/Nim/issues/4671>`_) +- Fixed "Generic procs accepting var .importcpp type do not work [regression]" + (`#4625 <https://github.com/nim-lang/Nim/issues/4625>`_) +- Fixed "C Error on tuple assignment with array" + (`#4626 <https://github.com/nim-lang/Nim/issues/4626>`_) +- Fixed "module securehash not gcsafe" + (`#4760 <https://github.com/nim-lang/Nim/issues/4760>`_) + +- Fixed "Nimble installation failed on Windows x86." + (`#4764 <https://github.com/nim-lang/Nim/issues/4764>`_) +- Fixed "Recent changes to marshal module break old marshalled data" + (`#4779 <https://github.com/nim-lang/Nim/issues/4779>`_) +- Fixed "tnewasyncudp.nim test loops forever" + (`#4777 <https://github.com/nim-lang/Nim/issues/4777>`_) +- Fixed "Wrong poll timeout behavior in asyncdispatch" + (`#4262 <https://github.com/nim-lang/Nim/issues/4262>`_) +- Fixed "Standalone await shouldn't read future" + (`#4170 <https://github.com/nim-lang/Nim/issues/4170>`_) +- Fixed "Regression: httpclient fails to compile without -d:ssl" + (`#4797 <https://github.com/nim-lang/Nim/issues/4797>`_) +- Fixed "C Error on declaring array of heritable objects with bitfields" + (`#3567 <https://github.com/nim-lang/Nim/issues/3567>`_) +- Fixed "Corruption when using Channels and Threads" + (`#4776 <https://github.com/nim-lang/Nim/issues/4776>`_) +- Fixed "Sometimes Channel tryRecv() erroneously reports no messages available on the first call on Windows" + (`#4746 <https://github.com/nim-lang/Nim/issues/4746>`_) +- Fixed "Improve error message of functions called without parenthesis" + (`#4813 <https://github.com/nim-lang/Nim/issues/4813>`_) +- Fixed "Docgen doesn't find doc comments in macro generated procs" + (`#4803 <https://github.com/nim-lang/Nim/issues/4803>`_) +- Fixed "asynchttpserver may consume unbounded memory reading headers" + (`#3847 <https://github.com/nim-lang/Nim/issues/3847>`_) +- Fixed "TLS connection to api.clashofclans.com hangs forever." + (`#4587 <https://github.com/nim-lang/Nim/issues/4587>`_) diff --git a/web/news/nim_community_survey_results.rst b/web/news/nim_community_survey_results.rst deleted file mode 100644 index 49656f20a..000000000 --- a/web/news/nim_community_survey_results.rst +++ /dev/null @@ -1,317 +0,0 @@ -Nim Community Survey Results -============================ - -.. container:: metadata - - Posted by Dominik Picheta on 20/08/2016 - -We have recently closed the 2016 Nim Community Survey. I am happy to -say that we have received exactly 790 responses, huge thanks go to the people -that took the time to respond. We're very thankful for this very valuable -feedback. - -This survey was inspired in part by the -`2016 State of Rust <https://blog.rust-lang.org/2016/06/30/State-of-Rust-Survey-2016.html>`_ -survey. You will note that many of the questions were modelled after -Rust's survey. One of the reasons for doing this was to allow us to easily -compare our results against the results obtained in the Rust survey. In -addition, we of course also liked many of their questions. - -Our survey ran from the 23rd of June 2016 until the 8th of August 2016. The -response numbers are impressive considering Nim's community size; at 790 they -make up just over 25% of the Rust survey's responses. - -The goal of this survey was to primarily determine how our community is using -Nim, in order to better understand how we should be improving it. In particular, -we wanted to know what people feel is missing from Nim in the lead up to -version 1.0. We have also asked our respondents about how well the Nim tools -worked, the challenges of adopting Nim, the resources that they used to learn -Nim and more. - -It is my hope that we will be able to run a similar survey in a years time, -doing so should give us an idea of whether we are improving. -With these general facts in mind, let's begin looking at specific questions. - -How did you find out about Nim? -------------------------------- - -The rationale for the first question was simple, we wanted to know where our -respondents found out about Nim. This is an interesting question for us, as -we do occassionally get users asking us why it took so long for them to hear -about Nim. It allows us to see how effective each website is at spreading the -word about Nim. - -.. raw::html - - <a href="../assets/news/images/survey/nim_found.png"> - <img src="../assets/news/images/survey/nim_found.png" alt="How did you find out about Nim?" style="width:100%"/> - </a> - -The majority of our respondents found Nim via Reddit, HackerNews or a search -engine such as Google. These results are not altogether surprising. There were -also a lot of "Other" responses, some of which were a bit more -interesting. These included multiple mentions of habrahabr.ru, Dr. Dobb's, -and lobste.rs. - -Do you use Nim? ---------------- - -Just like the Rust survey creators, we wanted to ensure that our survey was -open to both Nim users as well people who never used Nim. In addition to -those two groups, we have also included a third group of people: ex-Nim -users. All three are interesting, for many different reasons. -Nim users can tell us how they are using Nim and also how Nim's -tooling can improve. Ex-Nim users give us an -idea of why they stopped using Nim. Finally, respondents who never used Nim -can tell us the reasons for not adopting it. - -.. raw::html - - <a href="../assets/news/images/survey/do_you_use_nim.png"> - <img src="../assets/news/images/survey/do_you_use_nim.png" alt="Do you use Nim?" style="width:100%"/> - </a> - -It's nice to see that we have such a good range of respondents. The Rust survey -had a much larger number of Rust users amongst their respondents, with -no distinction between users that never used Rust and users that stopped using -Rust. - -.. raw::html - - <a href="https://blog.rust-lang.org/images/2016-06-Survey/do_you_use_rust.png"> - <img src="https://blog.rust-lang.org/images/2016-06-Survey/do_you_use_rust.png" alt="Do you use Rust?" style="width:100%"/> - </a> - -Should we consider your answers to be invalid? ----------------------------------------------- - -This was something I thought would be interesting to have, after I saw it -being used in another survey. While it does pinpoint possibly -invalid respondents, I have opted against filtering those out. Mainly because -that would require re-creating each of the charts generated by Google Forms -manually. - -.. raw::html - - <a href="../assets/news/images/survey/reliability.png"> - <img src="../assets/news/images/survey/reliability.png" alt="Should we consider your answers to be invalid?" style="width:100%"/> - </a> - -According to the responses to this question, around 94% of our responses -can be considered reliable. - -Nim users ---------- - -The following questions were answered only by the 38.9% of our respondents -who identified themselves as Nim users. - -How long have you been using Nim? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. raw::html - - <a href="../assets/news/images/survey/nim_time.png"> - <img src="../assets/news/images/survey/nim_time.png" alt="How long have you been using Nim?" style="width:100%"/> - </a> - -A large proportion of our Nim users were new. This is good news as it means that -our community is growing, with a large proportion of new Nim users that could -become long-term Nimians. In total, more than 35% of Nim users can be considered -new having used Nim for less than 3 months. With 18% of Nim users that can -be considered very new having used Nim for less than a month. -This could suggest that 18% of our users have only just found out about Nim in -the last week or so and have not yet got the chance to use it extensively. - -The high percentages of long term Nim users are encouraging. -They suggest -that many users are continuing to use Nim after making it through the first -few months. The sharp drop at 7-9 months is interesting, but may simply be -due to the fact that there were fewer newcomers during that period, or it -could be because our respondents are more likely to estimate that they have -been using Nim for a year or half a year rather than the awkward 7-9 months. - -.. raw::html - - <a href="../assets/news/images/survey/nim_time_rust.png"> - <img src="../assets/news/images/survey/nim_time_rust.png" alt="Time using Nim and Rust" style="width:100%"/> - </a> - -The results for Nim and Rust are actually remarkably similar. They both show a -drop at 7-9 months, although Rust's isn't as dramatic. Nim on the other hand -has a significantly higher percentage of new Nim users. - -Do you use Nim at work? -~~~~~~~~~~~~~~~~~~~~~~~ - -An important aspect of a language's adoption is whether it is being used for -"real" work. We wanted to know how many people are using Nim in their day -jobs and under what circumstances it is used. - -.. raw::html - - <a href="../assets/news/images/survey/nim_at_work.png"> - <img src="../assets/news/images/survey/nim_at_work.png" alt="Do you use Nim at work?" style="width:100%"/> - </a> - -While a vast majority of our users are not using Nim at work, more than 25% -of them are. It's encouraging to see such a high number already, even before -we have released version 1.0. In fact, this percentage is likely close to 30%, -because many of the "Other" responses mention using Nim for the likes of -internal tools or small scripts to help with the respondent's work. - -.. raw::html - - <a href="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png"> - <img src="https://blog.rust-lang.org/images/2016-06-Survey/rust_at_work.png" alt="Do you use Rust at work?" style="width:100%"/> - </a> - -Interestingly, a larger percentage of Nim users are using Nim at work than -Rust users. The sample sizes are of course vastly different, but it's still an -interesting result. Combined, nearly 1/5th of Rust users are using Rust -commercially whereas more than a quarter of Nim users are using Nim -commercially. - -Approximately how large are all the Nim projects that you work on? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Finding out how large the Nim projects worked on by Nim users are is also -very valuable. - -.. raw::html - - <a href="../assets/news/images/survey/project_size.png"> - <img src="../assets/news/images/survey/project_size.png" alt="Nim project size for all users" style="width:100%"/> - </a> - -This shows us that currently Nim is primarily being used for small scripts and -applications, with nearly 60% of the projects consisting of less than 1,000 -lines of code. This makes sense as many of our users are not using Nim -professionally, but are doing so in their spare time. - -.. raw::html - - <a href="../assets/news/images/survey/project_size_work.png"> - <img src="../assets/news/images/survey/project_size_work.png" alt="Nim project size for work users" style="width:100%"/> - </a> - -The numbers for part-time and full-time work users of Nim tell a different -story. Over 70% of the projects written by full-time users are between 10,001 -and 100,000 lines of code. Part-time users show a slightly different trend, -with many more small projects, the majority being between 1,000 and -10,000 lines of code. - -Overall it's good to see that there is a few large projects out there which are -composed of more than 100,000 lines of code. We expect to see the amount of -large projects to grow with time, especially with version 1.0 on the way. - -.. raw::html - - <a href="../assets/news/images/survey/project_size_nim_rust.png"> - <img src="../assets/news/images/survey/project_size_nim_rust.png" alt="Nim project size for work users (Nim vs. Rust)" style="width:100%"/> - </a> - -In comparison to Rust the proportion of project sizes for full-time users is -vastly different. This is likely due to our small sample size. Project sizes for -part-time users between Rust and Nim are somewhat similar, with differences of -around 10% for each project size. - -Do you plan to try to use Nim at work? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. raw::html - - <a href="../assets/news/images/survey/planning_to_use_at_work.png"> - <img src="../assets/news/images/survey/planning_to_use_at_work.png" alt="Planning to use Nim at work?" style="width:100%"/> - </a> - -It's also encouraging to see that over 50% of Nim users are planning to use -Nim at work! This is slightly more than Rust's 40% and should help Nim's -adoption into even more areas. - -Nim and its tools -~~~~~~~~~~~~~~~~~ - -In this section of the survey, we wanted to find out the tools that Nim -users are utilising when developing Nim applications. - -What editor(s) do you use when writing Nim? -___________________________________________ - -Programmers are very specific when it comes to their editor of choice, because -of that it's good to know which editor is most popular among our community. - -.. raw::html - - <a href="../assets/news/images/survey/editors.png"> - <img src="../assets/news/images/survey/editors.png" alt="Editors used by Nim users" style="width:100%"/> - </a> - -Looks like Vim is the winner with almost 30%. Followed by Sublime Text and -Visual Studio Code. Aporia, the Nim IDE, gets a respectable 15.5%. There was -also more than -17% of answers which included "Other" editors, such as: Notepad++, Geany, gedit, -and Kate. - -What operating system(s) do you compile for and run your Nim projects on? -_________________________________________________________________________ - -This question gave us information about the most popular target operating -systems, as well as some of the more obscure ones. We have asked this question -to find out the platforms on which Nim applications run on most frequently. - -.. raw::html - - <a href="../assets/news/images/survey/target_os.png"> - <img src="../assets/news/images/survey/target_os.png" alt="Target operating systems" style="width:100%"/> - </a> - -This question allowed multiple choices, so each percentage is out of the total -number of respondents for this question. For example, 80.7% of the -respondents selected "Linux" but only 26.6% selected OS X. - -This makes Linux by far the most popular target for Nim applications. -Some "Other" targets included: BSD (OpenBSD, FreeBSD), iOS, Android, and -JavaScript. -It's great to see Nim being used on such a wide variety of platforms. - -What operating system(s) do you develop Nim projects on? -________________________________________________________ - -With this question, we wanted to know what operating systems are used for -development. - -.. raw::html - - <a href="../assets/news/images/survey/dev_os.png"> - <img src="../assets/news/images/survey/dev_os.png" alt="Development operating systems" style="width:100%"/> - </a> - -This question also allowed multiple choices and ended up with very similar -results. - -You can see that Linux is also the most popular developmental -platform for Nim. But it's more popular as a target platform. - -Which version(s) of Nim do you use for your applications? -_________________________________________________________ - -.. raw::html - - <a href="../assets/news/images/survey/nim_versions.png"> - <img src="../assets/news/images/survey/nim_versions.png" alt="Version use" style="width:100%"/> - </a> - -At the time of this survey, version 0.14.2 was the latest stable release. -It's no wonder that it is the most commonly used release of Nim. It's good to -see that the older versions are not used as often. The high use of ``Git HEAD (devel)`` -(nightly builds) isn't surprising, Nim is still evolving rapidly and our -release schedule is not regular or frequent. - -Once we go past the 1.0 release, we expect to see much less use of the unstable -``devel`` branch. - - - - - diff --git a/web/news/version_0_15_released.rst b/web/news/version_0_15_released.rst deleted file mode 100644 index 1e35fb627..000000000 --- a/web/news/version_0_15_released.rst +++ /dev/null @@ -1,108 +0,0 @@ -Version 0.15.0 released -======================= - -.. container:: metadata - - Posted by Dominik Picheta on 17/09/2016 - -Some text here. - -Changes affecting backwards compatibility ------------------------------------------ - -- De-deprecated ``re.nim`` because we have too much code using it - and it got the basic API right. - -- ``split`` with ``set[char]`` as a delimiter in ``strutils.nim`` - no longer strips and splits characters out of the target string - by the entire set of characters. Instead, it now behaves in a - similar fashion to ``split`` with ``string`` and ``char`` - delimiters. Use ``splitWhitespace`` to get the old behaviour. -- The command invocation syntax will soon apply to open brackets - and curlies too. This means that code like ``a [i]`` will be - interpreted as ``a([i])`` and not as ``a[i]`` anymore. Likewise - ``f (a, b)`` means that the tuple ``(a, b)`` is passed to ``f``. - The compiler produces a warning for ``a [i]``:: - - Warning: a [b] will be parsed as command syntax; spacing is deprecated - - See `<https://github.com/nim-lang/Nim/issues/3898>`_ for the relevant - discussion. -- Overloading the special operators ``.``, ``.()``, ``.=``, ``()`` now - should be enabled via ``{.experimental.}``. -- ``immediate`` templates and macros are now deprecated. - Instead use ``untyped`` parameters. -- The metatype ``expr`` is deprecated. Use ``untyped`` instead. -- The metatype ``stmt`` is deprecated. Use ``typed`` instead. -- The compiler is now more picky when it comes to ``tuple`` types. The - following code used to compile, now it's rejected: - -.. code-block:: nim - - import tables - var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() - rocketaims["hi"] = {(-1.int8, 0.int8): 0.int64}.toTable() - -Instead be consistent in your tuple usage and use tuple names for tuples -that have tuple name: - -.. code-block:: nim - - import tables - var rocketaims = initOrderedTable[string, Table[tuple[k: int8, v: int8], int64] ]() - rocketaims["hi"] = {(k: -1.int8, v: 0.int8): 0.int64}.toTable() - -- Now when you compile console application for Windows, console output - encoding is automatically set to UTF-8. - -Library Additions ------------------ - -- Added ``readHeaderRow`` and ``rowEntry`` to ``parsecsv.nim`` to provide - a lightweight alternative to python's ``csv.DictReader``. -- Added ``setStdIoUnbuffered`` proc to ``system.nim`` to enable unbuffered I/O. - -- Added ``center`` and ``rsplit`` to ``strutils.nim`` to - provide similar Python functionality for Nim's strings. - -- Added ``isTitle``, ``title``, ``swapCase``, ``isUpper``, ``toUpper``, - ``isLower``, ``toLower``, ``isAlpha``, ``isSpace``, and ``capitalize`` - to ``unicode.nim`` to provide unicode aware case manipulation and case - testing. - -- Added a new module ``lib/pure/strmisc.nim`` to hold uncommon string - operations. Currently contains ``partition``, ``rpartition`` - and ``expandTabs``. - -- Split out ``walkFiles`` in ``os.nim`` to three separate procs in order - to make a clear distinction of functionality. ``walkPattern`` iterates - over both files and directories, while ``walkFiles`` now only iterates - over files and ``walkDirs`` only iterates over directories. - -Compiler Additions ------------------- - -- The ``-d/--define`` flag can now optionally take a value to be used - by code at compile time. - -Nimscript Additions -------------------- - -- Finally it's possible to dis/enable specific hints and warnings in - Nimscript via the procs ``warning`` and ``hint``. -- Nimscript exports a proc named ``patchFile`` which can be used to - patch modules or include files for different Nimble packages, including - the ``stdlib`` package. - - -Language Additions ------------------- - -- Added ``{.intdefine.}`` and ``{.strdefine.}`` macros to make use of - (optional) compile time defines. -- If the first statement is an ``import system`` statement then ``system`` - is not imported implicitly anymore. This allows for code like - ``import system except echo`` or ``from system import nil``. - -Bugfixes --------- diff --git a/web/sponsors.csv b/web/sponsors.csv index ac35b284c..0701575d5 100644 --- a/web/sponsors.csv +++ b/web/sponsors.csv @@ -1,31 +1,34 @@ logo, name, url, this_month, all_time, since, level -assets/bountysource/secondspectrum.png,Second Spectrum,http://www.secondspectrum.com/,250,750,"May 5, 2016",250 -assets/bountysource/xored.svg,"Xored Software, Inc.",http://xored.com/,250,500,"Jun 20, 2016",250 -,avsej,http://avsej.net,75,85,"Jun 10, 2016",75 -,shkolnick-kun,https://github.com/shkolnick-kun,75,75,"Jul 6, 2016",75 -,flyx,http://flyx.org,35,140,"Apr 7, 2016",75 -,endragor,https://github.com/endragor,25,100,"Apr 7, 2016",25 -,euantorano,http://euantorano.co.uk,25,50,"Jun 7, 2016",25 -,FedericoCeratto,http://firelet.net,25,100,"Apr 7, 2016",25 -,"Adrian Veith",,25,100,"Apr 20, 2016",25 -,xxlabaza,https://github.com/xxlabaza,25,45,"Jun 17, 2016",25 -,"Yuriy Glukhov",,25,100,"Apr 6, 2016",25 -,"Jonathan Arnett",,10,30,"May 20, 2016",10 -,"Oskari Timperi",,10,20,"Jun 8, 2016",10 -,zachaysan,http://venn.lc,10,20,"Jun 7, 2016",10 -,"Matthew Baulch",,10,20,"Jun 7, 2016",10 -,RationalG,https://github.com/RationalG,10,20,"Jun 17, 2016",10 -,btbytes,https://www.btbytes.com/,10,40,"Apr 6, 2016",10 -,niebaopeng,https://github.com/niebaopeng,10,30,"Apr 15, 2016",10 -,moigagoo,http://sloth-ci.com,5,15,"May 13, 2016",5 -,calind,http://calindon.net,5,10,"Jun 7, 2016",5 -,swalf,https://github.com/swalf,5,35,"May 9, 2016",5 -,johnnovak,http://www.johnnovak.net/,5,20,"Apr 29, 2016",5 -,RyanMarcus,http://rmarcus.info,5,5,"Jul 19, 2016",5 -,Blumenversand,https://blumenversender.com/,5,5,"Jul 21, 2016",5 -,lenzenmi,https://github.com/lenzenmi,5,5,"Jul 28, 2016",5 -,"Handojo Goenadi",,5,20,"Apr 19, 2016",5 -,"Date in Asia",,5,5,"Jul 30, 2016",5 -,"Matthew Newton",,5,20,"Apr 20, 2016",5 -,"Michael D. Sklaroff",,1,4,"Apr 27, 2016",1 -,"Svend Knudsen",,1,4,"Apr 11, 2016",1 +assets/bountysource/secondspectrum.png,Second Spectrum,http://www.secondspectrum.com/,250,1250,"May 5, 2016",250 +assets/bountysource/xored.svg,"Xored Software, Inc.",http://xored.com/,250,1000,"Jun 20, 2016",250 +,shkolnick-kun,https://github.com/shkolnick-kun,75,225,"Jul 6, 2016",75 +,flyx,http://flyx.org,35,210,"Apr 7, 2016",75 +,"Yuriy Glukhov",,25,150,"Apr 6, 2016",25 +,endragor,https://github.com/endragor,25,150,"Apr 7, 2016",25 +,FedericoCeratto,http://firelet.net,25,150,"Apr 7, 2016",25 +,"Adrian Veith",,25,150,"Apr 20, 2016",25 +,skunkiferous,https://github.com/skunkiferous,100,100,"Oct 2, 2016",150 +,euantorano,http://euantorano.co.uk,25,100,"Jun 7, 2016",25 +,xxlabaza,https://github.com/xxlabaza,25,95,"Jun 17, 2016",25 +,btbytes,https://www.btbytes.com/,10,60,"Apr 6, 2016",10 +,niebaopeng,https://github.com/niebaopeng,10,50,"Apr 15, 2016",10 +,"Jonathan Arnett",,10,50,"May 20, 2016",10 +,swalf,https://github.com/swalf,5,45,"May 9, 2016",5 +,zolern,https://github.com/zolern,10,40,"Apr 15, 2016",10 +,"pyloor ",https://schwarz-weiss.cc/,10,40,"May 16, 2016",10 +,zachaysan,http://venn.lc,10,40,"Jun 7, 2016",10 +,"Matthew Baulch",,10,40,"Jun 7, 2016",10 +,"Oskari Timperi",,10,40,"Jun 8, 2016",10 +,"Handojo Goenadi",,5,35,"Apr 19, 2016",5 +,"Matthew Newton",,5,30,"Apr 20, 2016",5 +,johnnovak,http://www.johnnovak.net/,5,30,"Apr 29, 2016",5 +,RyanMarcus,http://rmarcus.info,5,15,"Jul 19, 2016",5 +,lenzenmi,https://github.com/lenzenmi,5,15,"Jul 28, 2016",5 +,cpunion,https://github.com/cpunion,10,10,"Sep 9, 2016",10 +,pandada8,https://github.com/pandada8,5,10,"Aug 12, 2016",5 +,abeaumont,http://alfredobeaumont.org/blog,5,10,"Aug 12, 2016",5 +,"Svend Knudsen",,1,6,"Apr 11, 2016",1 +,"Michael D. Sklaroff",,1,6,"Apr 27, 2016",1 +,csoriano89,https://github.com/csoriano89,5,5,"Sep 7, 2016",5 +,nicck,,1,2,"Aug 9, 2016",1 +,campbellr,,1,1,"Sep 4, 2016",1 diff --git a/web/ticker.html b/web/ticker.html index ecbfb5e3f..a05ea8476 100644 --- a/web/ticker.html +++ b/web/ticker.html @@ -1,3 +1,13 @@ +<a class="news" href="$1news/2016_09_30_version_0_15_0_released.html"> + <h4>September 30, 2016</h4> + <p>Nim version 0.15.0 has been released!</p> +</a> + +<a class="news" href="$1news/2016_09_03_nim_community_survey_results.html"> + <h4>September 3, 2016</h4> + <p>Nim Community Survey results</p> +</a> + <a class="news" href="$1news/2016_08_06_bountysource_update_the_road_to_v10.html"> <h4>August 6, 2016</h4> <p>BountySource Update: The Road to v1.0</p> @@ -13,13 +23,4 @@ <p>Nim version 0.14.2 has been released!</p> </a> -<a class="news" href="$1news/2016_06_04_meet_our_bountysource_sponsors.html"> - <h4>June 04, 2016</h4> - <p>Meet our BountySource sponsors</p> -</a> - -<a class="news" href="$1news/2016_01_27_nim_in_action_is_now_available.html"> - <h4>January 27, 2016</h4> - <p>Nim in Action is now available!</p> -</a> <a href="$1news.html" class="blue">See All News...</a> |
