diff options
245 files changed, 7457 insertions, 1983 deletions
diff --git a/.gitignore b/.gitignore index 57b8a68d4..50fa9a431 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ tags install.sh deinstall.sh +doc/html/ doc/*.html doc/*.pdf doc/*.idx @@ -47,6 +48,7 @@ xcuserdata/ /testresults.json testament.db /csources +dist/ # Private directories and files (IDEs) .*/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 54b40dcd7..76c94c8e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,7 @@ deploy-windows: artifacts: paths: - build/*.exe + - build/*.zip expire_in: 1 week tags: - windows diff --git a/.travis.yml b/.travis.yml index ebf287502..a2ba41e12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,9 @@ script: - nim c koch - ./koch boot - ./koch boot -d:release - - nim e install_nimble.nims + - ./koch nimble - nim e tests/test_nimscript.nims - - nimble update - - nimble install zip + - nimble install zip -y - nimble install opengl - nimble install sdl1 - nimble install jester@#head diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100644 index 5d05ddbf8..000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -set -e -set -x - -if [ ! -e csources/.git ]; then - git clone --depth 1 https://github.com/nim-lang/csources.git csources -fi - -cd "csources" -sh build.sh -cd ".." - -./bin/nim c koch -./koch boot -d:release -./koch geninstall - -set +x - -echo -echo 'Install Nim using "./install.sh <dir>" or "sudo ./install.sh <dir>".' diff --git a/build_tools.sh b/build_tools.sh deleted file mode 100644 index 4d867b0ad..000000000 --- a/build_tools.sh +++ /dev/null @@ -1,3 +0,0 @@ -./bin/nim c --noNimblePath -p:compiler -o:./bin/nimble dist/nimble/src/nimble.nim -./bin/nim c --noNimblePath -p:compiler -o:./bin/nimsuggest dist/nimsuggest/nimsuggest.nim -./bin/nim c -o:./bin/nimgrep tools/nimgrep.nim diff --git a/ci/nsis_build.bat b/ci/nsis_build.bat index 657b2b90a..4e9d0d67c 100644 --- a/ci/nsis_build.bat +++ b/ci/nsis_build.bat @@ -21,11 +21,8 @@ cd web\upload move /y docs-%NIMVER%.zip download cd ..\.. -Rem Build .zip file: -rem koch csources -d:release -rem koch xz -d:release -rem move /y build\nim-%NIMVER%.zip web\upload\download - +Rem Build csources +koch csources -d:release || exit /b rem Grab C sources and nimsuggest git clone --depth 1 https://github.com/nim-lang/csources.git @@ -46,7 +43,7 @@ koch_temp nsis -d:release || exit /b koch_temp zip -d:release || exit /b dir build move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x32.exe || exit /b -move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b +move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x32.zip || exit /b ReM Build Win64 version: @@ -59,4 +56,4 @@ koch_temp boot -d:release || exit /b koch_temp nsis -d:release || exit /b koch_temp zip -d:release || exit /b move /y build\nim_%NIMVER%.exe build\nim-%NIMVER%_x64.exe || exit /b -move /y build\nim_%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b +move /y build\nim-%NIMVER%.zip build\nim-%NIMVER%_x64.zip || exit /b diff --git a/compiler.nimble b/compiler.nimble index 66d2ee7c2..d949aa754 100644 --- a/compiler.nimble +++ b/compiler.nimble @@ -1,6 +1,6 @@ [Package] name = "compiler" -version = "0.15.1" +version = "0.15.3" author = "Andreas Rumpf" description = "Compiler package providing the compiler sources as a library." license = "MIT" diff --git a/compiler/ast.nim b/compiler/ast.nim index d8939fc60..8f4acfc3b 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -317,6 +317,10 @@ type TTypeKind* = enum # order is important! # Don't forget to change hti.nim if you make a change here # XXX put this into an include file to avoid this issue! + # several types are no longer used (guess which), but a + # spot in the sequence is kept for backwards compatibility + # (apparently something with bootstrapping) + # if you need to add a type, they can apparently be reused tyNone, tyBool, tyChar, tyEmpty, tyArrayConstr, tyNil, tyExpr, tyStmt, tyTypeDesc, tyGenericInvocation, # ``T[a, b]`` for types to invoke @@ -345,9 +349,9 @@ type tyInt, tyInt8, tyInt16, tyInt32, tyInt64, # signed integers tyFloat, tyFloat32, tyFloat64, tyFloat128, tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64, - tyBigNum, - tyConst, tyMutable, tyVarargs, - tyIter, # unused + tyUnused0, tyUnused1, tyUnused2, + tyVarargs, + tyUnused, tyProxy # used as errornous type (for idetools) tyBuiltInTypeClass #\ @@ -1046,8 +1050,6 @@ proc newSym*(symKind: TSymKind, name: PIdent, owner: PSym, var emptyNode* = newNode(nkEmpty) # There is a single empty node that is shared! Do not overwrite it! -var anyGlobal* = newSym(skVar, getIdent("*"), nil, unknownLineInfo()) - proc isMetaType*(t: PType): bool = return t.kind in tyMetaTypes or (t.kind == tyStatic and t.n == nil) or @@ -1579,14 +1581,6 @@ proc skipStmtList*(n: PNode): PNode = else: result = n -proc createMagic*(name: string, m: TMagic): PSym = - result = newSym(skProc, getIdent(name), nil, unknownLineInfo()) - result.magic = m - -let - opNot* = createMagic("not", mNot) - opContains* = createMagic("contains", mInSet) - when false: proc containsNil*(n: PNode): bool = # only for debugging diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index be49ddc87..2761f888b 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -355,6 +355,14 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) = linefmt(p, cpsStmts, "$1 = $2;$n", rdLoc(dest), rdLoc(src)) else: internalError("genAssignment: " & $ty.kind) + if optMemTracker in p.options and dest.s in {OnHeap, OnUnknown}: + #writeStackTrace() + #echo p.currLineInfo, " requesting" + linefmt(p, cpsStmts, "#memTrackerWrite((void*)$1, $2, $3, $4);$n", + addrLoc(dest), rope getSize(dest.t), + makeCString(p.currLineInfo.toFullPath), + rope p.currLineInfo.safeLineNm) + proc genDeepCopy(p: BProc; dest, src: TLoc) = var ty = skipTypes(dest.t, abstractVarRange) case ty.kind @@ -592,9 +600,9 @@ proc genEqProc(p: BProc, e: PNode, d: var TLoc) = proc genIsNil(p: BProc, e: PNode, d: var TLoc) = let t = skipTypes(e.sons[1].typ, abstractRange) if t.kind == tyProc and t.callConv == ccClosure: - unaryExpr(p, e, d, "$1.ClPrc == 0") + unaryExpr(p, e, d, "($1.ClPrc == 0)") else: - unaryExpr(p, e, d, "$1 == 0") + unaryExpr(p, e, d, "($1 == 0)") proc unaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = const @@ -1946,6 +1954,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = d.s = OnStatic proc expr(p: BProc, n: PNode, d: var TLoc) = + p.currLineInfo = n.info case n.kind of nkSym: var sym = n.sym diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index eac734b3d..60ee0eaee 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -122,7 +122,7 @@ proc mapType(typ: PType): TCTypeKind = of tyOpenArray, tyArrayConstr, tyArray, tyVarargs: result = ctArray of tyObject, tyTuple: result = ctStruct of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyIter, tyTypeDesc: + tyTypeDesc: result = mapType(lastSon(typ)) of tyEnum: if firstOrd(typ) < 0: @@ -206,6 +206,10 @@ proc cacheGetType(tab: TIdTable, key: PType): Rope = # linear search is not necessary anymore: result = Rope(idTableGet(tab, key)) +proc addAbiCheck(m: BModule, t: PType, name: Rope) = + if isDefined("checkabi"): + addf(m.s[cfsTypeInfo], "NIM_CHECK_SIZE($1, $2);$n", [name, rope(getSize(t))]) + proc getTempName(m: BModule): Rope = result = m.tmpBase & rope(m.labels) inc m.labels @@ -267,6 +271,11 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope = result = getSimpleTypeDesc(m, lastSon typ) else: result = nil + if result != nil and typ.isImportedType(): + if cacheGetType(m.typeCache, typ) == nil: + idTablePut(m.typeCache, typ, result) + addAbiCheck(m, typ, result) + proc pushType(m: BModule, typ: PType) = add(m.typeStack, typ) @@ -656,6 +665,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = let foo = getTypeDescAux(m, t.sons[1], check) addf(m.s[cfsTypes], "typedef $1 $2[$3];$n", [foo, result, rope(n)]) + else: addAbiCheck(m, t, result) of tyObject, tyTuple: if isImportedCppType(t) and typ.kind == tyGenericInst: # for instantiated templates we do not go through the type cache as the @@ -701,7 +711,9 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = idTablePut(m.typeCache, t, result) # always call for sideeffects: let recdesc = if t.kind != tyTuple: getRecordDesc(m, t, result, check) else: getTupleDesc(m, t, result, check) - if not isImportedType(t): add(m.s[cfsTypes], recdesc) + if not isImportedType(t): + add(m.s[cfsTypes], recdesc) + elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result) of tySet: result = getTypeName(t.lastSon) & "Set" idTablePut(m.typeCache, t, result) @@ -711,8 +723,7 @@ proc getTypeDescAux(m: BModule, typ: PType, check: var IntSet): Rope = of 1, 2, 4, 8: addf(m.s[cfsTypes], "typedef NU$2 $1;$n", [result, rope(s*8)]) else: addf(m.s[cfsTypes], "typedef NU8 $1[$2];$n", [result, rope(getSize(t))]) - of tyGenericInst, tyDistinct, tyOrdinal, tyConst, tyMutable, - tyIter, tyTypeDesc: + of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc: result = getTypeDescAux(m, lastSon(t), check) else: internalError("getTypeDescAux(" & $t.kind & ')') diff --git a/compiler/ccgutils.nim b/compiler/ccgutils.nim index ecd98a2bf..2216cb4fd 100644 --- a/compiler/ccgutils.nim +++ b/compiler/ccgutils.nim @@ -93,7 +93,7 @@ proc getUniqueType*(key: PType): PType = # produced instead of ``NI``. result = key of tyEmpty, tyNil, tyExpr, tyStmt, tyPointer, tyString, - tyCString, tyNone, tyBigNum, tyVoid: + tyCString, tyNone, tyVoid: result = gCanonicalTypes[k] if result == nil: gCanonicalTypes[k] = key @@ -106,7 +106,7 @@ proc getUniqueType*(key: PType): PType = of tyDistinct: if key.deepCopy != nil: result = key else: result = getUniqueType(lastSon(key)) - of tyGenericInst, tyOrdinal, tyMutable, tyConst, tyIter, tyStatic: + of tyGenericInst, tyOrdinal, tyStatic: result = getUniqueType(lastSon(key)) #let obj = lastSon(key) #if obj.sym != nil and obj.sym.name.s == "TOption": @@ -153,6 +153,7 @@ proc getUniqueType*(key: PType): PType = else: # ugh, we need the canon here: result = slowSearch(key, k) + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("getUniqueType") proc tableGetType*(tab: TIdTable, key: PType): RootRef = # returns nil if we need to declare this type diff --git a/compiler/cgen.nim b/compiler/cgen.nim index d80a68609..6e18c8389 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,6 +16,8 @@ import condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, semparallel +from modulegraphs import ModuleGraph + import strutils except `%` # collides with ropes.`%` when options.hasTinyCBackend: @@ -870,7 +872,7 @@ proc genMainProc(m: BModule) = NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" & "$1" & "}$N$N" - + NimMainProc = "N_CDECL(void, NimMain)(void) {$N" & "\tvoid (*volatile inner)();$N" & @@ -972,7 +974,13 @@ proc getSomeInitName(m: PSym, suffix: string): Rope = result.add m.name.s result.add suffix -proc getInitName(m: PSym): Rope = getSomeInitName(m, "Init000") +proc getInitName(m: PSym): Rope = + if sfMainModule in m.flags: + # generate constant name for main module, for "easy" debugging. + result = rope"NimMainModule" + else: + result = getSomeInitName(m, "Init000") + proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") proc registerModuleToMain(m: PSym) = @@ -1168,7 +1176,7 @@ proc newModule(module: PSym): BModule = if (sfDeadCodeElim in module.flags): internalError("added pending module twice: " & module.filename) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = result = newModule(module) if optGenIndex in gGlobalOptions and generatedHeader == nil: let f = if headerFile.len > 0: headerFile else: gProjectFull @@ -1203,7 +1211,7 @@ proc getCFile(m: BModule): string = else: ".c" result = changeFileExt(completeCFilePath(m.cfilename.withPackageName), ext) -proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = +proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext = assert optSymbolFiles in gGlobalOptions var m = newModule(module) readMergeInfo(getCFile(m), m) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index a94950029..faeea7afb 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -68,6 +68,7 @@ type beforeRetNeeded*: bool # true iff 'BeforeRet' label for proc is needed threadVarAccessed*: bool # true if the proc already accessed some threadvar lastLineInfo*: TLineInfo # to avoid generating excessive 'nimln' statements + currLineInfo*: TLineInfo # AST codegen will make this superfluous nestedTryStmts*: seq[PNode] # in how many nested try statements we are # (the vars must be volatile then) inExceptBlock*: int # are we currently inside an except block? diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index bcf0b535b..5f0d71cc6 100644 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -165,8 +165,9 @@ proc methodDef*(s: PSym, fromCache: bool) = if witness.isNil: witness = gMethods[i].methods[0] # create a new dispatcher: add(gMethods, (methods: @[s], dispatcher: createDispatcher(s))) - if fromCache: - internalError(s.info, "no method dispatcher found") + #echo "adding ", s.info + #if fromCache: + # internalError(s.info, "no method dispatcher found") if witness != nil: localError(s.info, "invalid declaration order; cannot attach '" & s.name.s & "' to method defined here: " & $witness.info) diff --git a/compiler/commands.nim b/compiler/commands.nim index de1197292..590c4871d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -124,17 +124,17 @@ proc splitSwitch(switch: string, cmd, arg: var string, pass: TCmdLinePass, proc processOnOffSwitch(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = - case whichKeyword(arg) - of wOn: gOptions = gOptions + op - of wOff: gOptions = gOptions - op + case arg.normalize + of "on": gOptions = gOptions + op + of "off": gOptions = gOptions - op else: localError(info, errOnOrOffExpectedButXFound, arg) proc processOnOffSwitchOrList(op: TOptions, arg: string, pass: TCmdLinePass, info: TLineInfo): bool = result = false - case whichKeyword(arg) - of wOn: gOptions = gOptions + op - of wOff: gOptions = gOptions - op + case arg.normalize + of "on": gOptions = gOptions + op + of "off": gOptions = gOptions - op else: if arg == "list": result = true @@ -143,9 +143,9 @@ proc processOnOffSwitchOrList(op: TOptions, arg: string, pass: TCmdLinePass, proc processOnOffSwitchG(op: TGlobalOptions, arg: string, pass: TCmdLinePass, info: TLineInfo) = - case whichKeyword(arg) - of wOn: gGlobalOptions = gGlobalOptions + op - of wOff: gGlobalOptions = gGlobalOptions - op + case arg.normalize + of "on": gGlobalOptions = gGlobalOptions + op + of "off": gGlobalOptions = gGlobalOptions - op else: localError(info, errOnOrOffExpectedButXFound, arg) proc expectArg(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = @@ -178,12 +178,12 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass, var x = findStr(msgs.WarningsToStr, id) if x >= 0: n = TNoteKind(x + ord(warnMin)) else: localError(info, "unknown warning: " & id) - case whichKeyword(substr(arg, i)) - of wOn: + case substr(arg, i).normalize + of "on": incl(gNotes, n) incl(gMainPackageNotes, n) incl(enableNotes, n) - of wOff: + of "off": excl(gNotes, n) excl(gMainPackageNotes, n) incl(disableNotes, n) @@ -242,6 +242,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "linetrace": result = contains(gOptions, optLineTrace) of "debugger": result = contains(gOptions, optEndb) of "profiler": result = contains(gOptions, optProfiler) + of "memtracker": result = contains(gOptions, optMemTracker) of "checks", "x": result = gOptions * ChecksOptions == ChecksOptions of "floatchecks": result = gOptions * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck} @@ -264,6 +265,7 @@ proc testCompileOption*(switch: string, info: TLineInfo): bool = of "implicitstatic": result = contains(gOptions, optImplicitStatic) of "patterns": result = contains(gOptions, optPatterns) of "experimental": result = gExperimentalMode + of "excessivestacktrace": result = contains(gGlobalOptions, optExcessiveStackTrace) else: invalidCmdLineOption(passCmd1, switch, info) proc processPath(path: string, info: TLineInfo, @@ -445,6 +447,10 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = processOnOffSwitch({optProfiler}, arg, pass, info) if optProfiler in gOptions: defineSymbol("profiler") else: undefSymbol("profiler") + of "memtracker": + processOnOffSwitch({optMemTracker}, arg, pass, info) + if optMemTracker in gOptions: defineSymbol("memtracker") + else: undefSymbol("memtracker") of "checks", "x": processOnOffSwitch(ChecksOptions, arg, pass, info) of "floatchecks": processOnOffSwitch({optNaNCheck, optInfCheck}, arg, pass, info) @@ -630,12 +636,8 @@ proc processSwitch(switch, arg: string, pass: TCmdLinePass, info: TLineInfo) = of "dynliboverride": dynlibOverride(switch, arg, pass, info) of "cs": + # only supported for compatibility. Does nothing. expectArg(switch, arg, pass, info) - case arg - of "partial": idents.firstCharIsCS = true - of "none": idents.firstCharIsCS = false - else: localError(info, errGenerated, - "'partial' or 'none' expected, but found " & arg) of "experimental": expectNoArg(switch, arg, pass, info) gExperimentalMode = true diff --git a/compiler/depends.nim b/compiler/depends.nim index 1ccb134f2..9087f89f2 100644 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -12,6 +12,8 @@ import os, options, ast, astalgo, msgs, ropes, idents, passes, importer +from modulegraphs import ModuleGraph + proc generateDot*(project: string) type @@ -46,7 +48,7 @@ proc generateDot(project: string) = rope(changeFileExt(extractFilename(project), "")), gDotGraph], changeFileExt(project, "dot")) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/docgen.nim b/compiler/docgen.nim index c220902ff..76b36d796 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -184,7 +184,7 @@ proc genRecComment(d: PDoc, n: PNode): Rope = if n == nil: return nil result = genComment(d, n).rope if result == nil: - if n.kind notin {nkEmpty..nkNilLit, nkEnumTy}: + if n.kind notin {nkEmpty..nkNilLit, nkEnumTy, nkTupleTy}: for i in countup(0, len(n)-1): result = genRecComment(d, n.sons[i]) if result != nil: return @@ -514,7 +514,7 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind): JsonNode = result["code"] = %r.buf proc checkForFalse(n: PNode): bool = - result = n.kind == nkIdent and identEq(n.ident, "false") + result = n.kind == nkIdent and cmpIgnoreStyle(n.ident.s, "false") == 0 proc traceDeps(d: PDoc, n: PNode) = const k = skModule @@ -691,7 +691,7 @@ proc writeOutputJson*(d: PDoc, filename, outExt: string, discard "fixme: error report" proc commandDoc*() = - var ast = parseFile(gProjectMainIdx) + var ast = parseFile(gProjectMainIdx, newIdentCache()) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true @@ -721,7 +721,7 @@ proc commandRst2TeX*() = commandRstAux(gProjectFull, TexExt) proc commandJson*() = - var ast = parseFile(gProjectMainIdx) + var ast = parseFile(gProjectMainIdx, newIdentCache()) if ast == nil: return var d = newDocumentor(gProjectFull, options.gConfigVars) d.hasToc = true diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index d70d5406c..9504ab52f 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -13,6 +13,8 @@ import os, options, ast, astalgo, msgs, ropes, idents, passes, docgen +from modulegraphs import ModuleGraph + type TGen = object of TPassContext doc: PDoc @@ -49,7 +51,7 @@ proc processNodeJson(c: PPassContext, n: PNode): PNode = var g = PGen(c) generateJson(g.doc, n) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = var g: PGen new(g) g.module = module diff --git a/compiler/filter_tmpl.nim b/compiler/filter_tmpl.nim index 9e123e3a1..361b3d276 100644 --- a/compiler/filter_tmpl.nim +++ b/compiler/filter_tmpl.nim @@ -62,10 +62,7 @@ proc withInExpr(p: TTmplParser): bool {.inline.} = result = p.par > 0 or p.bracket > 0 or p.curly > 0 proc parseLine(p: var TTmplParser) = - var - d, j, curly: int - keyw: string - j = 0 + var j = 0 while p.x[j] == ' ': inc(j) if p.x[0] == p.nimDirective and p.x[1] == '?': newLine(p) @@ -73,16 +70,16 @@ proc parseLine(p: var TTmplParser) = newLine(p) inc(j) while p.x[j] == ' ': inc(j) - d = j - keyw = "" + let d = j + var keyw = "" while p.x[j] in PatternChars: add(keyw, p.x[j]) inc(j) scanPar(p, j) p.pendingExprLine = withInExpr(p) or llstream.endsWithOpr(p.x) - case whichKeyword(keyw) - of wEnd: + case keyw + of "end": if p.indent >= 2: dec(p.indent, 2) else: @@ -90,15 +87,15 @@ proc parseLine(p: var TTmplParser) = localError(p.info, errXNotAllowedHere, "end") llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, "#end") - of wIf, wWhen, wTry, wWhile, wFor, wBlock, wCase, wProc, wIterator, - wConverter, wMacro, wTemplate, wMethod: + of "if", "when", "try", "while", "for", "block", "case", "proc", "iterator", + "converter", "macro", "template", "method": llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, substr(p.x, d)) inc(p.indent, 2) - of wElif, wOf, wElse, wExcept, wFinally: + of "elif", "of", "else", "except", "finally": llStreamWrite(p.outp, spaces(p.indent - 2)) llStreamWrite(p.outp, substr(p.x, d)) - of wLet, wVar, wConst, wType: + of "wLet", "wVar", "wConst", "wType": llStreamWrite(p.outp, spaces(p.indent)) llStreamWrite(p.outp, substr(p.x, d)) if not p.x.contains({':', '='}): @@ -158,7 +155,7 @@ proc parseLine(p: var TTmplParser) = llStreamWrite(p.outp, p.toStr) llStreamWrite(p.outp, '(') inc(j) - curly = 0 + var curly = 0 while true: case p.x[j] of '\0': diff --git a/compiler/filters.nim b/compiler/filters.nim index adafe464e..d1a6409ff 100644 --- a/compiler/filters.nim +++ b/compiler/filters.nim @@ -30,7 +30,7 @@ proc getArg(n: PNode, name: string, pos: int): PNode = for i in countup(1, sonsLen(n) - 1): if n.sons[i].kind == nkExprEqExpr: if n.sons[i].sons[0].kind != nkIdent: invalidPragma(n) - if identEq(n.sons[i].sons[0].ident, name): + if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0: return n.sons[i].sons[1] elif i == pos: return n.sons[i] @@ -50,8 +50,8 @@ proc strArg(n: PNode, name: string, pos: int, default: string): string = proc boolArg(n: PNode, name: string, pos: int, default: bool): bool = var x = getArg(n, name, pos) if x == nil: result = default - elif (x.kind == nkIdent) and identEq(x.ident, "true"): result = true - elif (x.kind == nkIdent) and identEq(x.ident, "false"): result = false + elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "true") == 0: result = true + elif x.kind == nkIdent and cmpIgnoreStyle(x.ident.s, "false") == 0: result = false else: invalidPragma(n) proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream = @@ -62,7 +62,7 @@ proc filterStrip(stdin: PLLStream, filename: string, call: PNode): PLLStream = var line = newStringOfCap(80) while llStreamReadLine(stdin, line): var stripped = strip(line, leading, trailing) - if (len(pattern) == 0) or startsWith(stripped, pattern): + if len(pattern) == 0 or startsWith(stripped, pattern): llStreamWriteln(result, stripped) else: llStreamWriteln(result, line) diff --git a/compiler/idents.nim b/compiler/idents.nim index d9b72baf0..eecfa60a1 100644 --- a/compiler/idents.nim +++ b/compiler/idents.nim @@ -12,7 +12,7 @@ # id. This module is essential for the compiler's performance. import - hashes, strutils, etcpriv + hashes, strutils, etcpriv, wordrecg type TIdObj* = object of RootObj @@ -25,12 +25,20 @@ type next*: PIdent # for hash-table chaining h*: Hash # hash value of s -var firstCharIsCS*: bool = true -var buckets*: array[0..4096 * 2 - 1, PIdent] + IdentCache* = ref object + buckets: array[0..4096 * 2 - 1, PIdent] + wordCounter: int + idAnon*, idDelegator*, emptyIdent*: PIdent + +var + legacy: IdentCache + +proc resetIdentCache*() = + for i in low(legacy.buckets)..high(legacy.buckets): + legacy.buckets[i] = nil proc cmpIgnoreStyle(a, b: cstring, blen: int): int = - if firstCharIsCS: - if a[0] != b[0]: return 1 + if a[0] != b[0]: return 1 var i = 0 var j = 0 result = 1 @@ -65,9 +73,9 @@ proc cmpExact(a, b: cstring, blen: int): int = if result == 0: if a[i] != '\0': result = 1 -var wordCounter = 1 +{.this: self.} -proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent = +proc getIdent*(self: IdentCache; identifier: cstring, length: int, h: Hash): PIdent = var idx = h and high(buckets) result = buckets[idx] var last: PIdent = nil @@ -97,16 +105,33 @@ proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent = else: result.id = id -proc getIdent*(identifier: string): PIdent = +proc getIdent*(self: IdentCache; identifier: string): PIdent = result = getIdent(cstring(identifier), len(identifier), hashIgnoreStyle(identifier)) -proc getIdent*(identifier: string, h: Hash): PIdent = +proc getIdent*(self: IdentCache; identifier: string, h: Hash): PIdent = result = getIdent(cstring(identifier), len(identifier), h) -proc identEq*(id: PIdent, name: string): bool = - result = id.id == getIdent(name).id +proc newIdentCache*(): IdentCache = + if legacy.isNil: + result = IdentCache() + result.idAnon = result.getIdent":anonymous" + result.wordCounter = 1 + result.idDelegator = result.getIdent":delegator" + result.emptyIdent = result.getIdent("") + # initialize the keywords: + for s in countup(succ(low(specialWords)), high(specialWords)): + result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s) + legacy = result + else: + result = legacy -var idAnon* = getIdent":anonymous" -let idDelegator* = getIdent":delegator" +proc whichKeyword*(id: PIdent): TSpecialWord = + if id.id < 0: result = wInvalid + else: result = TSpecialWord(id.id) +proc getIdent*(identifier: string): PIdent = + ## for backwards compatibility. + if legacy.isNil: + discard newIdentCache() + legacy.getIdent identifier diff --git a/compiler/importer.nim b/compiler/importer.nim index 87415733b..feebf97c4 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -100,7 +100,7 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) = let ident = lookups.considerQuotedIdent(n) let s = strTableGet(fromMod.tab, ident) if s == nil: - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) else: if s.kind == skStub: loadStub(s) if s.kind notin ExportableSymKinds: @@ -162,12 +162,26 @@ proc importModuleAs(n: PNode, realModule: PSym): PSym = proc myImportModule(c: PContext, n: PNode): PSym = var f = checkModuleName(n) if f != InvalidFileIDX: - result = importModuleAs(n, gImportModule(c.module, f)) + let L = c.graph.importStack.len + let recursion = c.graph.importStack.find(f) + c.graph.importStack.add f + #echo "adding ", toFullPath(f), " at ", L+1 + if recursion >= 0: + var err = "" + for i in countup(recursion, L-1): + if i > recursion: err.add "\n" + err.add toFullPath(c.graph.importStack[i]) & " imports " & + toFullPath(c.graph.importStack[i+1]) + c.recursiveDep = err + result = importModuleAs(n, gImportModule(c.graph, c.module, f, c.cache)) + #echo "set back to ", L + c.graph.importStack.setLen(L) # we cannot perform this check reliably because of # test: modules/import_in_config) - if result.info.fileIndex == c.module.info.fileIndex and - result.info.fileIndex == n.info.fileIndex: - localError(n.info, errGenerated, "A module cannot import itself") + when true: + if result.info.fileIndex == c.module.info.fileIndex and + result.info.fileIndex == n.info.fileIndex: + localError(n.info, errGenerated, "A module cannot import itself") if sfDeprecated in result.flags: message(n.info, warnDeprecated, result.name.s) #suggestSym(n.info, result, false) diff --git a/compiler/installer.ini b/compiler/installer.ini index 2a9fa36a5..5e4b3ddbc 100644 --- a/compiler/installer.ini +++ b/compiler/installer.ini @@ -41,8 +41,8 @@ Files: "config/nimdoc.tex.cfg" ; Files: "doc/*.cfg" ; Files: "doc/*.pdf" ; Files: "doc/*.ini" -Files: "doc/overview.html" -Start: "doc/overview.html" +Files: "doc/html/overview.html" +Start: "doc/html/overview.html" [Other] @@ -63,6 +63,7 @@ Files: "icons/koch_icon.o" Files: "compiler" Files: "doc" +Files: "doc/html" Files: "tools" Files: "web/website.ini" Files: "web/ticker.html" @@ -89,12 +90,11 @@ Files: "bin/c2nim.exe" Files: "bin/nimgrep.exe" Files: "bin/nimsuggest.exe" Files: "bin/nimble.exe" -Files: "bin/makelink.exe" -Files: "bin/*.dll" Files: "koch.exe" +Files: "finish.exe" ; Files: "dist/mingw" -Files: "start.bat" +Files: r"tools\start.bat" BinPath: r"bin;dist\mingw\bin;dist" ; Section | dir | zipFile | size hint (in KB) | url | exe start menu entry @@ -104,6 +104,10 @@ Download: r"Support DLLs|bin|nim_dlls.zip|479|http://nim-lang.org/download/dlls. Download: r"Aporia Text Editor|dist|aporia.zip|97997|http://nim-lang.org/download/aporia-0.4.0.zip|aporia-0.4.0\bin\aporia.exe" ; for now only NSIS supports optional downloads +[WinBin] +Files: "$NIMINSTDEPS/makelink.exe" +Files: "$NIMINSTDEPS/*.dll" + [UnixBin] Files: "bin/nim" diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index e7fe8cc27..028dd00f0 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -35,6 +35,8 @@ import times, ropes, math, passes, ccgutils, wordrecg, renderer, rodread, rodutils, intsets, cgmeth, lowerings +from modulegraphs import ModuleGraph + type TTarget = enum targetJS, targetPHP @@ -138,7 +140,7 @@ proc declareGlobal(p: PProc; id: int; r: Rope) = const MappedToObject = {tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, - tySet, tyBigNum, tyVarargs} + tySet, tyVarargs} proc mapType(typ: PType): TJSTypeKind = let t = skipTypes(typ, abstractInst) @@ -151,15 +153,13 @@ proc mapType(typ: PType): TJSTypeKind = of tyPointer: # treat a tyPointer like a typed pointer to an array of bytes result = etyBaseIndex - of tyRange, tyDistinct, tyOrdinal, tyConst, tyMutable, tyIter, tyProxy: - result = mapType(t.sons[0]) + of tyRange, tyDistinct, tyOrdinal, tyProxy: result = mapType(t.sons[0]) of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt of tyBool: result = etyBool of tyFloat..tyFloat128: result = etyFloat of tySet: result = etyObject # map a set to a table of tyString, tySequence: result = etySeq - of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyBigNum, - tyVarargs: + of tyObject, tyArray, tyArrayConstr, tyTuple, tyOpenArray, tyVarargs: result = etyObject of tyNil: result = etyNull of tyGenericInst, tyGenericParam, tyGenericBody, tyGenericInvocation, @@ -171,6 +171,7 @@ proc mapType(typ: PType): TJSTypeKind = else: result = etyNone of tyProc: result = etyProc of tyCString: result = etyString + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapType") proc mapType(p: PProc; typ: PType): TJSTypeKind = if p.target == targetPHP: result = etyObject @@ -2273,11 +2274,11 @@ proc myClose(b: PPassContext, n: PNode): PNode = for obj, content in items(globals.classes): genClass(obj, content, ext) -proc myOpenCached(s: PSym, rd: PRodReader): PPassContext = +proc myOpenCached(graph: ModuleGraph; s: PSym, rd: PRodReader): PPassContext = internalError("symbol files are not possible with the JS code generator") result = nil -proc myOpen(s: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext = var r = newModule(s) r.target = if gCmd == cmdCompileToPHP: targetPHP else: targetJS result = r diff --git a/compiler/lexer.nim b/compiler/lexer.nim index 9c513034b..2769d757c 100644 --- a/compiler/lexer.nim +++ b/compiler/lexer.nim @@ -127,8 +127,9 @@ type # this is needed because scanning comments # needs so much look-ahead currLineIndent*: int - strongSpaces*: bool + strongSpaces*, allowTabs*: bool errorHandler*: TErrorHandler + cache*: IdentCache var gLinesCompiled*: int # all lines that have been compiled @@ -164,7 +165,6 @@ proc tokToStr*(tok: TToken): string = if tok.ident != nil: result = tok.ident.s else: - internalError("tokToStr") result = "" proc prettyTok*(tok: TToken): string = @@ -175,8 +175,6 @@ proc printTok*(tok: TToken) = msgWriteln($tok.line & ":" & $tok.col & "\t" & TokTypeToStr[tok.tokType] & " " & tokToStr(tok)) -var dummyIdent: PIdent - proc initToken*(L: var TToken) = L.tokType = tkInvalid L.iNumber = 0 @@ -185,7 +183,7 @@ proc initToken*(L: var TToken) = L.literal = "" L.fNumber = 0.0 L.base = base10 - L.ident = dummyIdent + L.ident = nil proc fillToken(L: var TToken) = L.tokType = tkInvalid @@ -195,17 +193,20 @@ proc fillToken(L: var TToken) = setLen(L.literal, 0) L.fNumber = 0.0 L.base = base10 - L.ident = dummyIdent + L.ident = nil -proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream) = +proc openLexer*(lex: var TLexer, fileIdx: int32, inputstream: PLLStream; + cache: IdentCache) = openBaseLexer(lex, inputstream) lex.fileIdx = fileidx lex.indentAhead = - 1 lex.currLineIndent = 0 inc(lex.lineNumber, inputstream.lineOffset) + lex.cache = cache -proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream) = - openLexer(lex, filename.fileInfoIdx, inputstream) +proc openLexer*(lex: var TLexer, filename: string, inputstream: PLLStream; + cache: IdentCache) = + openLexer(lex, filename.fileInfoIdx, inputstream, cache) proc closeLexer*(lex: var TLexer) = inc(gLinesCompiled, lex.lineNumber) @@ -746,7 +747,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = else: break h = !$h - tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) L.bufpos = pos if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)): @@ -757,7 +758,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) = proc endOperator(L: var TLexer, tok: var TToken, pos: int, hash: Hash) {.inline.} = var h = !$hash - tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) + tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h) if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon)) L.bufpos = pos @@ -847,34 +848,23 @@ proc scanComment(L: var TLexer, tok: var TToken) = tok.tokType = tkComment # iNumber contains the number of '\n' in the token tok.iNumber = 0 - when not defined(nimfix): - assert buf[pos+1] == '#' - if buf[pos+2] == '[': - skipMultiLineComment(L, tok, pos+3, true) - return - inc(pos, 2) + assert buf[pos+1] == '#' + if buf[pos+2] == '[': + skipMultiLineComment(L, tok, pos+3, true) + return + inc(pos, 2) var toStrip = 0 while buf[pos] == ' ': inc pos inc toStrip - when defined(nimfix): - var col = getColNumber(L, pos) while true: var lastBackslash = -1 while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: if buf[pos] == '\\': lastBackslash = pos+1 add(tok.literal, buf[pos]) inc(pos) - when defined(nimfix): - if lastBackslash > 0: - # a backslash is a continuation character if only followed by spaces - # plus a newline: - while buf[lastBackslash] == ' ': inc(lastBackslash) - if buf[lastBackslash] notin {CR, LF, nimlexbase.EndOfFile}: - # false positive: - lastBackslash = -1 pos = handleCRLF(L, pos) buf = L.buf @@ -883,21 +873,13 @@ proc scanComment(L: var TLexer, tok: var TToken) = inc(pos) inc(indent) - when defined(nimfix): - template doContinue(): untyped = - buf[pos] == '#' and (col == indent or lastBackslash > 0) - else: - template doContinue(): untyped = - buf[pos] == '#' and buf[pos+1] == '#' - if doContinue(): + if buf[pos] == '#' and buf[pos+1] == '#': tok.literal.add "\n" - when defined(nimfix): col = indent - else: - inc(pos, 2) - var c = toStrip - while buf[pos] == ' ' and c > 0: - inc pos - dec c + inc(pos, 2) + var c = toStrip + while buf[pos] == ' ' and c > 0: + inc pos + dec c inc tok.iNumber else: if buf[pos] > ' ': @@ -915,7 +897,7 @@ proc skip(L: var TLexer, tok: var TToken) = inc(pos) inc(tok.strongSpaceA) of '\t': - lexMessagePos(L, errTabulatorsAreNotAllowed, pos) + if not L.allowTabs: lexMessagePos(L, errTabulatorsAreNotAllowed, pos) inc(pos) of CR, LF: pos = handleCRLF(L, pos) @@ -932,27 +914,19 @@ proc skip(L: var TLexer, tok: var TToken) = else: break tok.strongSpaceA = 0 - when defined(nimfix): - template doBreak(): untyped = buf[pos] > ' ' - else: - template doBreak(): untyped = - buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#') - if doBreak(): + if buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#'): tok.indent = indent L.currLineIndent = indent break of '#': - when defined(nimfix): - break + # do not skip documentation comment: + if buf[pos+1] == '#': break + if buf[pos+1] == '[': + skipMultiLineComment(L, tok, pos+2, false) + pos = L.bufpos + buf = L.buf else: - # do not skip documentation comment: - if buf[pos+1] == '#': break - if buf[pos+1] == '[': - skipMultiLineComment(L, tok, pos+2, false) - pos = L.bufpos - buf = L.buf - else: - while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) + while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos) else: break # EndOfFile also leaves the loop L.bufpos = pos @@ -1051,7 +1025,7 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = if L.buf[L.bufpos] notin SymChars+{'_'} and not isMagicIdentSeparatorRune(L.buf, L.bufpos): tok.tokType = tkSymbol - tok.ident = getIdent("_") + tok.ident = L.cache.getIdent("_") else: tok.literal = $c tok.tokType = tkInvalid @@ -1084,5 +1058,3 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) = tok.tokType = tkInvalid lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') inc(L.bufpos) - -dummyIdent = getIdent("") diff --git a/compiler/lookups.nim b/compiler/lookups.nim index df19a6afb..fe159011c 100644 --- a/compiler/lookups.nim +++ b/compiler/lookups.nim @@ -242,6 +242,15 @@ proc errorUseQualifier*(c: PContext; info: TLineInfo; s: PSym) = inc i localError(info, errGenerated, err) +proc errorUndeclaredIdentifier*(c: PContext; info: TLineInfo; name: string) = + var err = "undeclared identifier: '" & name & "'" + if c.recursiveDep.len > 0: + err.add "\nThis might be caused by a recursive module dependency: " + err.add c.recursiveDep + # prevent excessive errors for 'nim check' + c.recursiveDep = nil + localError(info, errGenerated, err) + proc lookUp*(c: PContext, n: PNode): PSym = # Looks up a symbol. Generates an error in case of nil. case n.kind @@ -249,7 +258,7 @@ proc lookUp*(c: PContext, n: PNode): PSym = result = searchInScopes(c, n.ident).skipAlias(n) if result == nil: fixSpelling(n, n.ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, n.ident.s) + errorUndeclaredIdentifier(c, n.info, n.ident.s) result = errorSym(c, n) of nkSym: result = n.sym @@ -258,7 +267,7 @@ proc lookUp*(c: PContext, n: PNode): PSym = result = searchInScopes(c, ident).skipAlias(n) if result == nil: fixSpelling(n, ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) result = errorSym(c, n) else: internalError(n.info, "lookUp") @@ -282,7 +291,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = result = searchInScopes(c, ident, allExceptModule).skipAlias(n) if result == nil and checkUndeclared in flags: fixSpelling(n, ident, searchInScopes) - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) result = errorSym(c, n) elif checkAmbiguity in flags and result != nil and contains(c.ambiguousSymbols, result.id): @@ -307,7 +316,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym = result = strTableGet(m.tab, ident).skipAlias(n) if result == nil and checkUndeclared in flags: fixSpelling(n.sons[1], ident, searchInScopes) - localError(n.sons[1].info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.sons[1].info, ident.s) result = errorSym(c, n.sons[1]) elif n.sons[1].kind == nkSym: result = n.sons[1].sym diff --git a/compiler/lowerings.nim b/compiler/lowerings.nim index 9db4383f6..6a8eccb83 100644 --- a/compiler/lowerings.nim +++ b/compiler/lowerings.nim @@ -284,7 +284,7 @@ proc addLocalVar(varSection, varInit: PNode; owner: PSym; typ: PType; varInit.add newFastAsgnStmt(newSymNode(result), v) else: let deepCopyCall = newNodeI(nkCall, varInit.info, 3) - deepCopyCall.sons[0] = newSymNode(createMagic("deepCopy", mDeepCopy)) + deepCopyCall.sons[0] = newSymNode(getSysMagic("deepCopy", mDeepCopy)) deepCopyCall.sons[1] = newSymNode(result) deepCopyCall.sons[2] = v varInit.add deepCopyCall @@ -356,7 +356,7 @@ proc createWrapperProc(f: PNode; threadParam, argsParam: PSym; if fk == fvGC: "data" else: "blob", fv.info), call) if fk == fvGC: let incRefCall = newNodeI(nkCall, fv.info, 2) - incRefCall.sons[0] = newSymNode(createMagic("GCref", mGCref)) + incRefCall.sons[0] = newSymNode(getSysMagic("GCref", mGCref)) incRefCall.sons[1] = indirectAccess(threadLocalProm.newSymNode, "data", fv.info) body.add incRefCall @@ -446,7 +446,7 @@ proc genHigh*(n: PNode): PNode = else: result = newNodeI(nkCall, n.info, 2) result.typ = getSysType(tyInt) - result.sons[0] = newSymNode(createMagic("high", mHigh)) + result.sons[0] = newSymNode(getSysMagic("high", mHigh)) result.sons[1] = n proc setupArgsForParallelism(n: PNode; objType: PType; scratchObj: PSym; diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index 13b365d04..6a9d69082 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -38,6 +38,14 @@ proc getSysSym*(name: string): PSym = if result.kind == skStub: loadStub(result) if result.kind == skAlias: result = result.owner +proc createMagic*(name: string, m: TMagic): PSym = + result = newSym(skProc, getIdent(name), nil, unknownLineInfo()) + result.magic = m + +let + opNot* = createMagic("not", mNot) + opContains* = createMagic("contains", mInSet) + proc getSysMagic*(name: string, m: TMagic): PSym = var ti: TIdentIter let id = getIdent(name) @@ -46,7 +54,7 @@ proc getSysMagic*(name: string, m: TMagic): PSym = if r.kind == skStub: loadStub(r) if r.magic == m: # prefer the tyInt variant: - if r.typ.sons[0].kind == tyInt: return r + if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r result = r r = nextIdentIter(ti, systemModule.tab) if result != nil: return result diff --git a/compiler/main.nim b/compiler/main.nim index 0db66b53e..2118078be 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -15,7 +15,8 @@ import wordrecg, sem, semdata, idents, passes, docgen, extccomp, cgen, jsgen, json, nversion, platform, nimconf, importer, passaux, depends, vm, vmdef, types, idgen, - docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists + docgen2, service, parser, modules, ccgutils, sigmatch, ropes, lists, + modulegraphs from magicsys import systemModule, resetSysTypes @@ -30,80 +31,44 @@ proc semanticPasses = registerPass verbosePass registerPass semPass -proc commandGenDepend = +proc commandGenDepend(graph: ModuleGraph; cache: IdentCache) = semanticPasses() registerPass(gendependPass) registerPass(cleanupPass) - compileProject() + compileProject(graph, cache) generateDot(gProjectFull) execExternalProgram("dot -Tpng -o" & changeFileExt(gProjectFull, "png") & ' ' & changeFileExt(gProjectFull, "dot")) -proc commandCheck = +proc commandCheck(graph: ModuleGraph; cache: IdentCache) = msgs.gErrorMax = high(int) # do not stop after first error defineSymbol("nimcheck") semanticPasses() # use an empty backend for semantic checking only rodPass() - compileProject() + compileProject(graph, cache) -proc commandDoc2(json: bool) = +proc commandDoc2(graph: ModuleGraph; cache: IdentCache; json: bool) = msgs.gErrorMax = high(int) # do not stop after first error semanticPasses() if json: registerPass(docgen2JsonPass) else: registerPass(docgen2Pass) #registerPass(cleanupPass()) - compileProject() + compileProject(graph, cache) finishDoc2Pass(gProjectName) -proc commandCompileToC = +proc commandCompileToC(graph: ModuleGraph; cache: IdentCache) = extccomp.initVars() semanticPasses() registerPass(cgenPass) rodPass() #registerPass(cleanupPass()) - compileProject() + compileProject(graph, cache) cgenWriteModules() if gCmd != cmdRun: extccomp.callCCompiler(changeFileExt(gProjectFull, "")) - if isServing: - # caas will keep track only of the compilation commands - lastCaasCmd = curCaasCmd - resetCgenModules() - for i in 0 .. <gMemCacheData.len: - gMemCacheData[i].hashStatus = hashCached - gMemCacheData[i].needsRecompile = Maybe - - # XXX: clean these global vars - # ccgstmts.gBreakpoints - # ccgthreadvars.nimtv - # ccgthreadvars.nimtVDeps - # ccgthreadvars.nimtvDeclared - # cgendata - # cgmeth? - # condsyms? - # depends? - # lexer.gLinesCompiled - # msgs - error counts - # magicsys, when system.nim changes - # rodread.rodcompilerProcs - # rodread.gTypeTable - # rodread.gMods - - # !! ropes.cache - # semthreads.computed? - # - # suggest.usageSym - # - # XXX: can we run out of IDs? - # XXX: detect config reloading (implement as error/require restart) - # XXX: options are appended (they will accumulate over time) - resetCompilationLists() - ccgutils.resetCaches() - GC_fullCollect() - -proc commandCompileToJS = +proc commandCompileToJS(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) setTarget(osJS, cpuJS) #initDefines() @@ -113,9 +78,9 @@ proc commandCompileToJS = if gCmd == cmdCompileToPHP: defineSymbol("nimphp") semanticPasses() registerPass(JSgenPass) - compileProject() + compileProject(graph, cache) -proc interactivePasses = +proc interactivePasses(graph: ModuleGraph; cache: IdentCache) = #incl(gGlobalOptions, optSafeCode) #setTarget(osNimrodVM, cpuNimrodVM) initDefines() @@ -125,30 +90,30 @@ proc interactivePasses = registerPass(semPass) registerPass(evalPass) -proc commandInteractive = +proc commandInteractive(graph: ModuleGraph; cache: IdentCache) = msgs.gErrorMax = high(int) # do not stop after first error - interactivePasses() - compileSystemModule() + interactivePasses(graph, cache) + compileSystemModule(graph, cache) if commandArgs.len > 0: - discard compileModule(fileInfoIdx(gProjectFull), {}) + discard graph.compileModule(fileInfoIdx(gProjectFull), cache, {}) else: - var m = makeStdinModule() + var m = graph.makeStdinModule() incl(m.flags, sfMainModule) - processModule(m, llStreamOpenStdIn(), nil) + processModule(graph, m, llStreamOpenStdIn(), nil, cache) const evalPasses = [verbosePass, semPass, evalPass] -proc evalNim(nodes: PNode, module: PSym) = - carryPasses(nodes, module, evalPasses) +proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym; cache: IdentCache) = + carryPasses(graph, nodes, module, cache, evalPasses) -proc commandEval(exp: string) = +proc commandEval(graph: ModuleGraph; cache: IdentCache; exp: string) = if systemModule == nil: - interactivePasses() - compileSystemModule() - var echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")" - evalNim(echoExp.parseString, makeStdinModule()) + interactivePasses(graph, cache) + compileSystemModule(graph, cache) + let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")" + evalNim(graph, echoExp.parseString(cache), makeStdinModule(graph), cache) -proc commandScan = +proc commandScan(cache: IdentCache) = var f = addFileExt(mainCommandArg(), NimExt) var stream = llStreamOpen(f, fmRead) if stream != nil: @@ -156,7 +121,7 @@ proc commandScan = L: TLexer tok: TToken initToken(tok) - openLexer(L, f, stream) + openLexer(L, f, stream, cache) while true: rawGetTok(L, tok) printTok(tok) @@ -165,77 +130,11 @@ proc commandScan = else: rawMessage(errCannotOpenFile, f) -proc commandSuggest = - if isServing: - # XXX: hacky work-around ahead - # Currently, it's possible to issue a idetools command, before - # issuing the first compile command. This will leave the compiler - # cache in a state where "no recompilation is necessary", but the - # cgen pass was never executed at all. - commandCompileToC() - let gDirtyBufferIdx = gTrackPos.fileIndex - discard compileModule(gDirtyBufferIdx, {sfDirty}) - resetModule(gDirtyBufferIdx) - else: - msgs.gErrorMax = high(int) # do not stop after first error - semanticPasses() - rodPass() - # XXX: this handles the case when the dirty buffer is the main file, - # but doesn't handle the case when it's imported module - #var projFile = if gProjectMainIdx == gDirtyOriginalIdx: gDirtyBufferIdx - # else: gProjectMainIdx - compileProject() #(projFile) - -proc resetMemory = - resetCompilationLists() - ccgutils.resetCaches() - resetAllModules() - resetRopeCache() - resetSysTypes() - gOwners = @[] - for i in low(buckets)..high(buckets): - buckets[i] = nil - idAnon = nil - - # XXX: clean these global vars - # ccgstmts.gBreakpoints - # ccgthreadvars.nimtv - # ccgthreadvars.nimtVDeps - # ccgthreadvars.nimtvDeclared - # cgendata - # cgmeth? - # condsyms? - # depends? - # lexer.gLinesCompiled - # msgs - error counts - # magicsys, when system.nim changes - # rodread.rodcompilerProcs - # rodread.gTypeTable - # rodread.gMods - - # !! ropes.cache - # - # suggest.usageSym - # - # XXX: can we run out of IDs? - # XXX: detect config reloading (implement as error/require restart) - # XXX: options are appended (they will accumulate over time) - # vis = visimpl - when compileOption("gc", "v2"): - gcDebugging = true - echo "COLLECT 1" - GC_fullCollect() - echo "COLLECT 2" - GC_fullCollect() - echo "COLLECT 3" - GC_fullCollect() - echo GC_getStatistics() - const SimulateCaasMemReset = false PrintRopeCacheStats = false -proc mainCommand* = +proc mainCommand*(graph: ModuleGraph; cache: IdentCache) = when SimulateCaasMemReset: gGlobalOptions.incl(optCaasEnabled) @@ -251,66 +150,66 @@ proc mainCommand* = of "c", "cc", "compile", "compiletoc": # compile means compileToC currently gCmd = cmdCompileToC - commandCompileToC() + commandCompileToC(graph, cache) of "cpp", "compiletocpp": gCmd = cmdCompileToCpp defineSymbol("cpp") - commandCompileToC() + commandCompileToC(graph, cache) of "objc", "compiletooc": gCmd = cmdCompileToOC defineSymbol("objc") - commandCompileToC() + commandCompileToC(graph, cache) of "run": gCmd = cmdRun when hasTinyCBackend: extccomp.setCC("tcc") - commandCompileToC() + commandCompileToC(graph, cache) else: rawMessage(errInvalidCommandX, command) of "js", "compiletojs": gCmd = cmdCompileToJS - commandCompileToJS() + commandCompileToJS(graph, cache) of "php": gCmd = cmdCompileToPHP - commandCompileToJS() + commandCompileToJS(graph, cache) of "doc": wantMainModule() gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandDoc() of "doc2": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) defineSymbol("nimdoc") - commandDoc2(false) + commandDoc2(graph, cache, false) of "rst2html": gCmd = cmdRst2html - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandRst2Html() of "rst2tex": gCmd = cmdRst2tex - loadConfigs(DocTexConfig) + loadConfigs(DocTexConfig, cache) commandRst2TeX() of "jsondoc": wantMainModule() gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) wantMainModule() defineSymbol("nimdoc") commandJson() of "jsondoc2": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) wantMainModule() defineSymbol("nimdoc") - commandDoc2(true) + commandDoc2(graph, cache, true) of "buildindex": gCmd = cmdDoc - loadConfigs(DocConfig) + loadConfigs(DocConfig, cache) commandBuildIndex() of "gendepend": gCmd = cmdGenDepend - commandGenDepend() + commandGenDepend(graph, cache) of "dump": gCmd = cmdDump if getConfigVar("dump.format") == "json": @@ -339,35 +238,21 @@ proc mainCommand* = for it in iterSearchPath(searchPaths): msgWriteln(it) of "check": gCmd = cmdCheck - commandCheck() + commandCheck(graph, cache) of "parse": gCmd = cmdParse wantMainModule() - discard parseFile(gProjectMainIdx) + discard parseFile(gProjectMainIdx, cache) of "scan": gCmd = cmdScan wantMainModule() - commandScan() - msgWriteln("Beware: Indentation tokens depend on the parser\'s state!") + commandScan(cache) + msgWriteln("Beware: Indentation tokens depend on the parser's state!") of "secret": gCmd = cmdInteractive - commandInteractive() + commandInteractive(graph, cache) of "e": - # XXX: temporary command for easier testing - commandEval(mainCommandArg()) - of "reset": - resetMemory() - of "idetools": - gCmd = cmdIdeTools - if gEvalExpr != "": - commandEval(gEvalExpr) - else: - commandSuggest() - of "serve": - isServing = true - gGlobalOptions.incl(optCaasEnabled) - msgs.gErrorMax = high(int) # do not stop after first error - serve(mainCommand) + commandEval(graph, cache, mainCommandArg()) of "nop", "help": # prevent the "success" message: gCmd = cmdDump @@ -394,3 +279,5 @@ proc mainCommand* = resetMemory() resetAttributes() + +proc mainCommand*() = mainCommand(newModuleGraph(), newIdentCache()) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim new file mode 100644 index 000000000..38fd4f89f --- /dev/null +++ b/compiler/modulegraphs.nim @@ -0,0 +1,112 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements the module graph data structure. The module graph +## represents a complete Nim project. Single modules can either be kept in RAM +## or stored in a ROD file. The ROD file mechanism is not yet integrated here. +## +## The caching of modules is critical for 'nimsuggest' and is tricky to get +## right. If module E is being edited, we need autocompletion (and type +## checking) for E but we don't want to recompile depending +## modules right away for faster turnaround times. Instead we mark the module's +## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we +## need to recompile it and all of its dependencies that are marked as 'dirty'. +## 'nimsuggest sug' actually is invoked for the file being edited so we know +## its content changed and there is no need to compute any checksums. +## Instead of a recursive algorithm, we use an iterative algorithm: +## +## - If a module gets recompiled, its dependencies need to be updated. +## - Its dependent module stays the same. +## + +import ast, intsets, tables + +type + ModuleGraph* = ref object + modules*: seq[PSym] ## indexed by int32 fileIdx + packageSyms*: TStrTable + deps*: IntSet # the dependency graph or potentially its transitive closure. + suggestMode*: bool # whether we are in nimsuggest mode or not. + invalidTransitiveClosure: bool + inclToMod*: Table[int32, int32] # mapping of include file to the + # first module that included it + importStack*: seq[int32] # The current import stack. Used for detecting recursive + # module dependencies. + +{.this: g.} + +proc newModuleGraph*(): ModuleGraph = + result = ModuleGraph() + initStrTable(result.packageSyms) + result.deps = initIntSet() + result.modules = @[] + result.importStack = @[] + result.inclToMod = initTable[int32, int32]() + +proc resetAllModules*(g: ModuleGraph) = + initStrTable(packageSyms) + deps = initIntSet() + modules = @[] + importStack = @[] + inclToMod = initTable[int32, int32]() + +proc getModule*(g: ModuleGraph; fileIdx: int32): PSym = + if fileIdx >= 0 and fileIdx < modules.len: + result = modules[fileIdx] + +proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b + +proc addDep*(g: ModuleGraph; m: PSym, dep: int32) = + if suggestMode: + deps.incl m.position.dependsOn(dep) + # we compute the transitive closure later when quering the graph lazily. + # this improve efficiency quite a lot: + invalidTransitiveClosure = true + +proc addIncludeDep*(g: ModuleGraph; module, includeFile: int32) = + discard hasKeyOrPut(inclToMod, includeFile, module) + +proc parentModule*(g: ModuleGraph; fileIdx: int32): int32 = + ## returns 'fileIdx' if the file belonging to this index is + ## directly used as a module or else the module that first + ## references this include file. + if fileIdx >= 0 and fileIdx < modules.len and modules[fileIdx] != nil: + result = fileIdx + else: + result = inclToMod.getOrDefault(fileIdx) + +proc transitiveClosure(g: var IntSet; n: int) = + # warshall's algorithm + for k in 0..<n: + for i in 0..<n: + for j in 0..<n: + if i != j and not g.contains(i.dependsOn(j)): + if g.contains(i.dependsOn(k)) and g.contains(k.dependsOn(j)): + g.incl i.dependsOn(j) + +proc markDirty*(g: ModuleGraph; fileIdx: int32) = + let m = getModule fileIdx + if m != nil: incl m.flags, sfDirty + +proc markClientsDirty*(g: ModuleGraph; fileIdx: int32) = + # we need to mark its dependent modules D as dirty right away because after + # nimsuggest is done with this module, the module's dirty flag will be + # cleared but D still needs to be remembered as 'dirty'. + if invalidTransitiveClosure: + invalidTransitiveClosure = false + transitiveClosure(deps, modules.len) + + # every module that *depends* on this file is also dirty: + for i in 0i32..<modules.len.int32: + let m = modules[i] + if m != nil and deps.contains(i.dependsOn(fileIdx)): + incl m.flags, sfDirty + +proc isDirty*(g: ModuleGraph; m: PSym): bool = + result = suggestMode and sfDirty in m.flags diff --git a/compiler/modules.nim b/compiler/modules.nim index 711fb6aa4..3451d85ec 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -7,130 +7,121 @@ # distribution, for details about the copyright. # -## implements the module handling +## Implements the module handling, including the caching of modules. import ast, astalgo, magicsys, securehash, rodread, msgs, cgendata, sigmatch, options, - idents, os, lexer, idgen, passes, syntaxes, llstream - -type - TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled - THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged - - TModuleInMemory* = object - compiledAt*: float - hash*: SecureHash - deps*: seq[int32] ## XXX: slurped files are currently not tracked - needsRecompile*: TNeedRecompile - hashStatus*: THashStatus - -var - gCompiledModules: seq[PSym] = @[] - gMemCacheData*: seq[TModuleInMemory] = @[] - ## XXX: we should implement recycling of file IDs - ## if the user keeps renaming modules, the file IDs will keep growing - gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. - packageSyms: TStrTable - -initStrTable(packageSyms) - -proc getModule*(fileIdx: int32): PSym = - if fileIdx >= 0 and fileIdx < gCompiledModules.len: - result = gCompiledModules[fileIdx] - -proc hashChanged(fileIdx: int32): bool = - internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len - - template updateStatus = - gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged - else: hashNotChanged - # echo "TESTING Hash: ", fileIdx.toFilename, " ", result - - case gMemCacheData[fileIdx].hashStatus - of hashHasChanged: - result = true - of hashNotChanged: - result = false - of hashCached: - let newHash = secureHashFile(fileIdx.toFullPath) - result = newHash != gMemCacheData[fileIdx].hash - gMemCacheData[fileIdx].hash = newHash - updateStatus() - of hashNotTaken: - gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) - result = true - updateStatus() - -proc doHash(fileIdx: int32) = - if gMemCacheData[fileIdx].hashStatus == hashNotTaken: - # echo "FIRST Hash: ", fileIdx.ToFilename - gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) - -proc addDep(x: PSym, dep: int32) = - growCache gMemCacheData, dep - gMemCacheData[x.position].deps.safeAdd(dep) - -proc resetModule*(fileIdx: int32) = - # echo "HARD RESETTING ", fileIdx.toFilename - if fileIdx <% gMemCacheData.len: - gMemCacheData[fileIdx].needsRecompile = Yes - if fileIdx <% gCompiledModules.len: - gCompiledModules[fileIdx] = nil - if fileIdx <% cgendata.gModules.len: - cgendata.gModules[fileIdx] = nil - -proc resetModule*(module: PSym) = - let conflict = getModule(module.position.int32) - if conflict == nil: return - doAssert conflict == module - resetModule(module.position.int32) - initStrTable(module.tab) - -proc resetAllModules* = - for i in 0..gCompiledModules.high: - if gCompiledModules[i] != nil: - resetModule(i.int32) - resetPackageCache() - initStrTable(packageSyms) - # for m in cgenModules(): echo "CGEN MODULE FOUND" - -proc resetAllModulesHard* = - resetPackageCache() - gCompiledModules.setLen 0 - gMemCacheData.setLen 0 - magicsys.resetSysTypes() - initStrTable(packageSyms) - # XXX - #gOwners = @[] - -proc checkDepMem(fileIdx: int32): TNeedRecompile = - template markDirty = - resetModule(fileIdx) - return Yes - - if gFuzzyGraphChecking: - if gMemCacheData[fileIdx].needsRecompile != Maybe: - return gMemCacheData[fileIdx].needsRecompile - else: - # cycle detection: We claim that a cycle does no harm. - if gMemCacheData[fileIdx].needsRecompile == Probing: - return No + idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs + +when false: + type + TNeedRecompile* = enum Maybe, No, Yes, Probing, Recompiled + THashStatus* = enum hashNotTaken, hashCached, hashHasChanged, hashNotChanged + + TModuleInMemory* = object + hash*: SecureHash + deps*: seq[int32] ## XXX: slurped files are currently not tracked + + needsRecompile*: TNeedRecompile + hashStatus*: THashStatus + + var + gCompiledModules: seq[PSym] = @[] + gMemCacheData*: seq[TModuleInMemory] = @[] + ## XXX: we should implement recycling of file IDs + ## if the user keeps renaming modules, the file IDs will keep growing + gFuzzyGraphChecking*: bool # nimsuggest uses this. XXX figure out why. + + proc hashChanged(fileIdx: int32): bool = + internalAssert fileIdx >= 0 and fileIdx < gMemCacheData.len + + template updateStatus = + gMemCacheData[fileIdx].hashStatus = if result: hashHasChanged + else: hashNotChanged + # echo "TESTING Hash: ", fileIdx.toFilename, " ", result + + case gMemCacheData[fileIdx].hashStatus + of hashHasChanged: + result = true + of hashNotChanged: + result = false + of hashCached: + let newHash = secureHashFile(fileIdx.toFullPath) + result = newHash != gMemCacheData[fileIdx].hash + gMemCacheData[fileIdx].hash = newHash + updateStatus() + of hashNotTaken: + gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) + result = true + updateStatus() + + proc doHash(fileIdx: int32) = + if gMemCacheData[fileIdx].hashStatus == hashNotTaken: + # echo "FIRST Hash: ", fileIdx.ToFilename + gMemCacheData[fileIdx].hash = secureHashFile(fileIdx.toFullPath) + + proc resetModule*(fileIdx: int32) = + # echo "HARD RESETTING ", fileIdx.toFilename + if fileIdx <% gMemCacheData.len: + gMemCacheData[fileIdx].needsRecompile = Yes + if fileIdx <% gCompiledModules.len: + gCompiledModules[fileIdx] = nil + if fileIdx <% cgendata.gModules.len: + cgendata.gModules[fileIdx] = nil + + proc resetModule*(module: PSym) = + let conflict = getModule(module.position.int32) + if conflict == nil: return + doAssert conflict == module + resetModule(module.position.int32) + initStrTable(module.tab) + + proc resetAllModules* = + for i in 0..gCompiledModules.high: + if gCompiledModules[i] != nil: + resetModule(i.int32) + resetPackageCache() + # for m in cgenModules(): echo "CGEN MODULE FOUND" + + proc resetAllModulesHard* = + resetPackageCache() + gCompiledModules.setLen 0 + gMemCacheData.setLen 0 + magicsys.resetSysTypes() + # XXX + #gOwners = @[] + + proc checkDepMem(fileIdx: int32): TNeedRecompile = + template markDirty = + resetModule(fileIdx) + return Yes + + if gFuzzyGraphChecking: + if gMemCacheData[fileIdx].needsRecompile != Maybe: + return gMemCacheData[fileIdx].needsRecompile + else: + # cycle detection: We claim that a cycle does no harm. + if gMemCacheData[fileIdx].needsRecompile == Probing: + return No + + if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): + markDirty() - if optForceFullMake in gGlobalOptions or hashChanged(fileIdx): - markDirty() + if gMemCacheData[fileIdx].deps != nil: + gMemCacheData[fileIdx].needsRecompile = Probing + for dep in gMemCacheData[fileIdx].deps: + let d = checkDepMem(dep) + if d in {Yes, Recompiled}: + # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d + markDirty() - if gMemCacheData[fileIdx].deps != nil: - gMemCacheData[fileIdx].needsRecompile = Probing - for dep in gMemCacheData[fileIdx].deps: - let d = checkDepMem(dep) - if d in {Yes, Recompiled}: - # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d - markDirty() + gMemCacheData[fileIdx].needsRecompile = No + return No - gMemCacheData[fileIdx].needsRecompile = No - return No +proc resetSystemArtifacts*() = + magicsys.resetSysTypes() -proc newModule(fileIdx: int32): PSym = +proc newModule(graph: ModuleGraph; fileIdx: int32): PSym = # We cannot call ``newSym`` here, because we have to circumvent the ID # mechanism, which we do in order to assign each module a persistent ID. new(result) @@ -143,20 +134,19 @@ proc newModule(fileIdx: int32): PSym = result.info = newLineInfo(fileIdx, 1, 1) let pack = getIdent(getPackageName(filename)) - var packSym = packageSyms.strTableGet(pack) + var packSym = graph.packageSyms.strTableGet(pack) if packSym == nil: let pck = getPackageName(filename) let pck2 = if pck.len > 0: pck else: "unknown" packSym = newSym(skPackage, getIdent(pck2), nil, result.info) initStrTable(packSym.tab) - packageSyms.strTableAdd(packSym) + graph.packageSyms.strTableAdd(packSym) result.owner = packSym result.position = fileIdx - growCache gMemCacheData, fileIdx - growCache gCompiledModules, fileIdx - gCompiledModules[result.position] = result + growCache graph.modules, fileIdx + graph.modules[result.position] = result incl(result.flags, sfUsed) initStrTable(result.tab) @@ -167,57 +157,66 @@ proc newModule(fileIdx: int32): PSym = # strTableIncl() for error corrections: discard strTableIncl(packSym.tab, result) -proc compileModule*(fileIdx: int32, flags: TSymFlags): PSym = - result = getModule(fileIdx) +proc compileModule*(graph: ModuleGraph; fileIdx: int32; cache: IdentCache, flags: TSymFlags): PSym = + result = graph.getModule(fileIdx) if result == nil: - growCache gMemCacheData, fileIdx - gMemCacheData[fileIdx].needsRecompile = Probing - result = newModule(fileIdx) - #var rd = handleSymbolFile(result) + #growCache gMemCacheData, fileIdx + #gMemCacheData[fileIdx].needsRecompile = Probing + result = newModule(graph, fileIdx) var rd: PRodReader result.flags = result.flags + flags if sfMainModule in result.flags: gMainPackageId = result.owner.id if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: - rd = handleSymbolFile(result) + rd = handleSymbolFile(result, cache) if result.id < 0: - internalError("handleSymbolFile should have set the module\'s ID") + internalError("handleSymbolFile should have set the module's ID") return else: result.id = getID() - let validFile = processModule(result, if sfMainModule in flags and gProjectIsStdin: llStreamOpen(stdin) else: nil, rd) - if optCaasEnabled in gGlobalOptions: - gMemCacheData[fileIdx].compiledAt = gLastCmdTime - gMemCacheData[fileIdx].needsRecompile = Recompiled - if validFile: doHash fileIdx - else: - if checkDepMem(fileIdx) == Yes: - result = compileModule(fileIdx, flags) - else: - result = gCompiledModules[fileIdx] - -proc importModule*(s: PSym, fileIdx: int32): PSym {.procvar.} = + discard processModule(graph, result, + if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, + rd, cache) + #if optCaasEnabled in gGlobalOptions: + # gMemCacheData[fileIdx].needsRecompile = Recompiled + # if validFile: doHash fileIdx + elif graph.isDirty(result): + result.flags.excl sfDirty + # reset module fields: + initStrTable(result.tab) + result.ast = nil + discard processModule(graph, result, + if sfMainModule in flags and gProjectIsStdin: stdin.llStreamOpen else: nil, + nil, cache) + graph.markClientsDirty(fileIdx) + when false: + if checkDepMem(fileIdx) == Yes: + result = compileModule(fileIdx, cache, flags) + else: + result = gCompiledModules[fileIdx] + +proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: int32; + cache: IdentCache): PSym {.procvar.} = # this is called by the semantic checking phase - result = compileModule(fileIdx, {}) - if optCaasEnabled in gGlobalOptions: addDep(s, fileIdx) + result = compileModule(graph, fileIdx, cache, {}) + graph.addDep(s, fileIdx) #if sfSystemModule in result.flags: # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: gNotes = if s.owner.id == gMainPackageId: gMainPackageNotes else: ForeignPackageNotes -proc includeModule*(s: PSym, fileIdx: int32): PNode {.procvar.} = - result = syntaxes.parseFile(fileIdx) - if optCaasEnabled in gGlobalOptions: - growCache gMemCacheData, fileIdx - addDep(s, fileIdx) - doHash(fileIdx) +proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: int32; + cache: IdentCache): PNode {.procvar.} = + result = syntaxes.parseFile(fileIdx, cache) + graph.addDep(s, fileIdx) + graph.addIncludeDep(s.position.int32, fileIdx) -proc compileSystemModule* = +proc compileSystemModule*(graph: ModuleGraph; cache: IdentCache) = if magicsys.systemModule == nil: systemFileIdx = fileInfoIdx(options.libpath/"system.nim") - discard compileModule(systemFileIdx, {sfSystemModule}) + discard graph.compileModule(systemFileIdx, cache, {sfSystemModule}) proc wantMainModule* = if gProjectFull.len == 0: @@ -227,18 +226,20 @@ proc wantMainModule* = passes.gIncludeFile = includeModule passes.gImportModule = importModule -proc compileProject*(projectFileIdx = -1'i32) = +proc compileProject*(graph: ModuleGraph; cache: IdentCache; + projectFileIdx = -1'i32) = wantMainModule() let systemFileIdx = fileInfoIdx(options.libpath / "system.nim") let projectFile = if projectFileIdx < 0: gProjectMainIdx else: projectFileIdx + graph.importStack.add projectFile if projectFile == systemFileIdx: - discard compileModule(projectFile, {sfMainModule, sfSystemModule}) + discard graph.compileModule(projectFile, cache, {sfMainModule, sfSystemModule}) else: - compileSystemModule() - discard compileModule(projectFile, {sfMainModule}) + graph.compileSystemModule(cache) + discard graph.compileModule(projectFile, cache, {sfMainModule}) -proc makeModule*(filename: string): PSym = - result = newModule(fileInfoIdx filename) +proc makeModule*(graph: ModuleGraph; filename: string): PSym = + result = graph.newModule(fileInfoIdx filename) result.id = getID() -proc makeStdinModule*(): PSym = makeModule"stdin" +proc makeStdinModule*(graph: ModuleGraph): PSym = graph.makeModule"stdin" diff --git a/compiler/msgs.nim b/compiler/msgs.nim index fd0aafccb..94b0bee00 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -35,7 +35,7 @@ type errNoneSpeedOrSizeExpectedButXFound, errGuiConsoleOrLibExpectedButXFound, errUnknownOS, errUnknownCPU, errGenOutExpectedButXFound, errArgsNeedRunOption, errInvalidMultipleAsgn, errColonOrEqualsExpected, - errExprExpected, errUndeclaredIdentifier, errUndeclaredField, + errExprExpected, errUndeclaredField, errUndeclaredRoutine, errUseQualifier, errTypeExpected, errSystemNeeds, errExecutionOfProgramFailed, errNotOverloadable, @@ -197,7 +197,6 @@ const errInvalidMultipleAsgn: "multiple assignment is not allowed", errColonOrEqualsExpected: "\':\' or \'=\' expected, but found \'$1\'", errExprExpected: "expression expected, but found \'$1\'", - errUndeclaredIdentifier: "undeclared identifier: \'$1\'", errUndeclaredField: "undeclared field: \'$1\'", errUndeclaredRoutine: "attempting to call undeclared routine: \'$1\'", errUseQualifier: "ambiguous identifier: \'$1\' -- use a qualifier", @@ -215,7 +214,7 @@ const errOrdinalTypeExpected: "ordinal type expected", errOrdinalOrFloatTypeExpected: "ordinal or float type expected", errOverOrUnderflow: "over- or underflow", - errCannotEvalXBecauseIncompletelyDefined: "cannot evalutate '$1' because type is not defined completely", + errCannotEvalXBecauseIncompletelyDefined: "cannot evaluate '$1' because type is not defined completely", errChrExpectsRange0_255: "\'chr\' expects an int in the range 0..255", errDynlibRequiresExportc: "\'dynlib\' requires \'exportc\'", errUndeclaredFieldX: "undeclared field: \'$1\'", @@ -676,9 +675,8 @@ proc getInfoContext*(index: int): TLineInfo = if i >=% L: result = unknownLineInfo() else: result = msgContext[i] -proc toFilename*(fileIdx: int32): string = - if fileIdx < 0: result = "???" - else: result = fileInfos[fileIdx].projPath +template toFilename*(fileIdx: int32): string = + (if fileIdx < 0: "???" else: fileInfos[fileIdx].projPath) proc toFullPath*(fileIdx: int32): string = if fileIdx < 0: result = "???" diff --git a/compiler/nim.nim b/compiler/nim.nim index a58afd593..35afecf20 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -13,10 +13,15 @@ when defined(gcc) and defined(windows): else: {.link: "icons/nim_icon.o".} +when defined(amd64) and defined(windows) and defined(vcc): + {.link: "icons/nim-amd64-windows-vcc.res".} +when defined(i386) and defined(windows) and defined(vcc): + {.link: "icons/nim-i386-windows-vcc.res".} + import commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, extccomp, strutils, os, osproc, platform, main, parseopt, service, - nodejs, scriptconfig + nodejs, scriptconfig, idents, modulegraphs when hasTinyCBackend: import tccgen @@ -32,7 +37,7 @@ proc prependCurDir(f: string): string = else: result = f -proc handleCmdLine() = +proc handleCmdLine(cache: IdentCache) = if paramCount() == 0: writeCommandLineUsage() else: @@ -41,7 +46,7 @@ proc handleCmdLine() = if gProjectName == "-": gProjectName = "stdinfile" gProjectFull = "stdinfile" - gProjectPath = getCurrentDir() + gProjectPath = canonicalizePath getCurrentDir() gProjectIsStdin = true elif gProjectName != "": try: @@ -49,26 +54,26 @@ proc handleCmdLine() = except OSError: gProjectFull = gProjectName let p = splitFile(gProjectFull) - gProjectPath = p.dir + gProjectPath = canonicalizePath p.dir gProjectName = p.name else: - gProjectPath = getCurrentDir() + gProjectPath = canonicalizePath getCurrentDir() loadConfigs(DefaultConfig) # load all config files let scriptFile = gProjectFull.changeFileExt("nims") if fileExists(scriptFile): - runNimScript(scriptFile, freshDefines=false) + runNimScript(cache, scriptFile, freshDefines=false) # 'nim foo.nims' means to just run the NimScript file and do nothing more: if scriptFile == gProjectFull: return elif fileExists(gProjectPath / "config.nims"): # directory wide NimScript file - runNimScript(gProjectPath / "config.nims", freshDefines=false) + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) # now process command line arguments again, because some options in the # command line can overwite the config file's settings extccomp.initVars() processCmdLine(passCmd2, "") if options.command == "": rawMessage(errNoCommand, command) - mainCommand() + mainCommand(newModuleGraph(), cache) if optHints in gOptions and hintGCStats in gNotes: echo(GC_getStatistics()) #echo(GC_getStatistics()) if msgs.gErrorCounter == 0: @@ -112,5 +117,5 @@ when compileOption("gc", "v2") or compileOption("gc", "refc"): condsyms.initDefines() when not defined(selftest): - handleCmdLine() + handleCmdLine(newIdentCache()) msgQuit(int8(msgs.gErrorCounter > 0)) diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim index 496bd0123..4bf2fbc9a 100644 --- a/compiler/nimconf.nim +++ b/compiler/nimconf.nim @@ -156,7 +156,7 @@ proc checkSymbol(L: TLexer, tok: TToken) = lexMessage(L, errIdentifierExpected, tokToStr(tok)) proc parseAssignment(L: var TLexer, tok: var TToken) = - if tok.ident.id == getIdent("-").id or tok.ident.id == getIdent("--").id: + if tok.ident.s == "-" or tok.ident.s == "--": confTok(L, tok) # skip unnecessary prefix var info = getLineInfo(L, tok) # save for later in case of an error checkSymbol(L, tok) @@ -179,14 +179,14 @@ proc parseAssignment(L: var TLexer, tok: var TToken) = if tok.tokType == tkBracketRi: confTok(L, tok) else: lexMessage(L, errTokenExpected, "']'") add(val, ']') - let percent = tok.ident.id == getIdent("%=").id + let percent = tok.ident != nil and tok.ident.s == "%=" if tok.tokType in {tkColon, tkEquals} or percent: if len(val) > 0: add(val, ':') confTok(L, tok) # skip ':' or '=' or '%' checkSymbol(L, tok) add(val, tokToStr(tok)) confTok(L, tok) # skip symbol - while tok.ident != nil and tok.ident.id == getIdent("&").id: + while tok.ident != nil and tok.ident.s == "&": confTok(L, tok) checkSymbol(L, tok) add(val, tokToStr(tok)) @@ -197,7 +197,7 @@ proc parseAssignment(L: var TLexer, tok: var TToken) = else: processSwitch(s, val, passPP, info) -proc readConfigFile(filename: string) = +proc readConfigFile(filename: string; cache: IdentCache) = var L: TLexer tok: TToken @@ -205,7 +205,7 @@ proc readConfigFile(filename: string) = stream = llStreamOpen(filename, fmRead) if stream != nil: initToken(tok) - openLexer(L, filename, stream) + openLexer(L, filename, stream, cache) tok.tokType = tkEof # to avoid a pointless warning confTok(L, tok) # read in the first token while tok.tokType != tkEof: parseAssignment(L, tok) @@ -225,22 +225,22 @@ proc getSystemConfigPath(filename: string): string = if not existsFile(result): result = joinPath([p, "etc", filename]) if not existsFile(result): result = "/etc/" & filename -proc loadConfigs*(cfg: string) = +proc loadConfigs*(cfg: string; cache: IdentCache) = setDefaultLibpath() if optSkipConfigFile notin gGlobalOptions: - readConfigFile(getSystemConfigPath(cfg)) + readConfigFile(getSystemConfigPath(cfg), cache) if optSkipUserConfigFile notin gGlobalOptions: - readConfigFile(getUserConfigPath(cfg)) + readConfigFile(getUserConfigPath(cfg), cache) var pd = if gProjectPath.len > 0: gProjectPath else: getCurrentDir() if optSkipParentConfigFiles notin gGlobalOptions: for dir in parentDirs(pd, fromRoot=true, inclusive=false): - readConfigFile(dir / cfg) + readConfigFile(dir / cfg, cache) if optSkipProjConfigFile notin gGlobalOptions: - readConfigFile(pd / cfg) + readConfigFile(pd / cfg, cache) if gProjectName.len != 0: # new project wide config file: @@ -251,4 +251,8 @@ proc loadConfigs*(cfg: string) = projectConfig = changeFileExt(gProjectFull, "nimrod.cfg") if fileExists(projectConfig): rawMessage(warnDeprecated, projectConfig) - readConfigFile(projectConfig) + readConfigFile(projectConfig, cache) + +proc loadConfigs*(cfg: string) = + # for backwards compatibility only. + loadConfigs(cfg, newIdentCache()) diff --git a/compiler/nimeval.nim b/compiler/nimeval.nim index 2bddb76e7..2872bdade 100644 --- a/compiler/nimeval.nim +++ b/compiler/nimeval.nim @@ -8,10 +8,9 @@ # ## exposes the Nim VM to clients. - import ast, modules, passes, passaux, condsyms, - options, nimconf, lists, sem, semdata, llstream, vm + options, nimconf, lists, sem, semdata, llstream, vm, modulegraphs, idents proc execute*(program: string) = passes.gIncludeFile = includeModule @@ -27,7 +26,9 @@ proc execute*(program: string) = registerPass(evalPass) appendStr(searchPaths, options.libpath) - compileSystemModule() - var m = makeStdinModule() + var graph = newModuleGraph() + var cache = newIdentCache() + var m = makeStdinModule(graph) incl(m.flags, sfMainModule) - processModule(m, llStreamOpen(program), nil) + compileSystemModule(graph,cache) + processModule(graph,m, llStreamOpen(program), nil, cache) diff --git a/compiler/nimfix/nimfix.nim b/compiler/nimfix/nimfix.nim index 39436702f..b4007cdaf 100644 --- a/compiler/nimfix/nimfix.nim +++ b/compiler/nimfix/nimfix.nim @@ -73,7 +73,7 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = of "auto": gStyleCheck = StyleCheck.Auto else: localError(gCmdLineInfo, errOnOrOffExpected) of "wholeproject": gOnlyMainfile = false - of "besteffort": msgs.gErrorMax = high(int) # dont stop after first error + of "besteffort": msgs.gErrorMax = high(int) # don't stop after first error else: processSwitch(pass, p) of cmdArgument: diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim deleted file mode 100644 index 2be368d68..000000000 --- a/compiler/nimsuggest/nimsuggest.nim +++ /dev/null @@ -1,12 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nimsuggest has been moved to https://github.com/nim-lang/nimsuggest - -{.error: "This project has moved to the following repo: https://github.com/nim-lang/nimsuggest".} diff --git a/compiler/nversion.nim b/compiler/nversion.nim index d69e1e553..4d4fe6c95 100644 --- a/compiler/nversion.nim +++ b/compiler/nversion.nim @@ -13,5 +13,5 @@ const MaxSetElements* = 1 shl 16 # (2^16) to support unicode character sets? VersionAsString* = system.NimVersion - RodFileVersion* = "1221" # modify this if the rod-format changes! + RodFileVersion* = "1222" # modify this if the rod-format changes! diff --git a/compiler/options.nim b/compiler/options.nim index 7cf707945..9edafb17a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -34,7 +34,8 @@ type # please make sure we have under 32 options optProfiler, # profiler turned on optImplicitStatic, # optimization: implicit at compile time # evaluation - optPatterns # en/disable pattern matching + optPatterns, # en/disable pattern matching + optMemTracker TOptions* = set[TOption] TGlobalOption* = enum # **keep binary compatible** @@ -231,10 +232,10 @@ proc canonicalizePath*(path: string): string = proc shortenDir*(dir: string): string = ## returns the interesting part of a dir - var prefix = getPrefixDir() & DirSep + var prefix = gProjectPath & DirSep if startsWith(dir, prefix): return substr(dir, len(prefix)) - prefix = gProjectPath & DirSep + prefix = getPrefixDir() & DirSep if startsWith(dir, prefix): return substr(dir, len(prefix)) result = dir diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index f8f1f355c..c51d406ac 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -230,6 +230,8 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult # builtin slice keeps lvalue-ness: if getMagic(n) in {mArrGet, mSlice}: result = isAssignable(owner, n.sons[1], isUnsafeAddr) + elif n.typ != nil and n.typ.kind == tyVar: + result = arLValue of nkStmtList, nkStmtListExpr: if n.typ != nil: result = isAssignable(owner, n.lastSon, isUnsafeAddr) diff --git a/compiler/parser.nim b/compiler/parser.nim index 40862eb63..902bf0fcb 100644 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -28,15 +28,15 @@ import llstream, lexer, idents, strutils, ast, astalgo, msgs type - TParser*{.final.} = object # A TParser object represents a module that - # is being parsed - currInd: int # current indentation level + TParser*{.final.} = object # A TParser object represents a file that + # is being parsed + currInd: int # current indentation level firstTok, strongSpaces: bool # Has the first token been read? # Is strongSpaces on? - lex*: TLexer # The lexer that is used for parsing - tok*: TToken # The current token - inPragma: int # Pragma level - inSemiStmtList: int + lex*: TLexer # The lexer that is used for parsing + tok*: TToken # The current token + inPragma*: int # Pragma level + inSemiStmtList*: int proc parseAll*(p: var TParser): PNode proc closeParser*(p: var TParser) @@ -73,18 +73,20 @@ proc getTok(p: var TParser) = rawGetTok(p.lex, p.tok) proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream, + cache: IdentCache; strongSpaces=false) = ## Open a parser, using the given arguments to set up its internal state. ## initToken(p.tok) - openLexer(p.lex, fileIdx, inputStream) + openLexer(p.lex, fileIdx, inputStream, cache) getTok(p) # read the first token p.firstTok = true p.strongSpaces = strongSpaces proc openParser*(p: var TParser, filename: string, inputStream: PLLStream, + cache: IdentCache; strongSpaces=false) = - openParser(p, filename.fileInfoIdx, inputStream, strongSpaces) + openParser(p, filename.fileInfoIdx, inputStream, cache, strongSpaces) proc closeParser(p: var TParser) = ## Close a parser, freeing up its resources. @@ -320,9 +322,9 @@ proc parseSymbol(p: var TParser, allowNil = false): PNode = tkParLe..tkParDotRi}: accm.add(tokToStr(p.tok)) getTok(p) - result.add(newIdentNodeP(getIdent(accm), p)) + result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p)) of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: - result.add(newIdentNodeP(getIdent(tokToStr(p.tok)), p)) + result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p)) getTok(p) else: parMessage(p, errIdentifierExpected, p.tok) @@ -923,7 +925,7 @@ proc parseParamList(p: var TParser, retColon = true): PNode = optPar(p) eat(p, tkParRi) let hasRet = if retColon: p.tok.tokType == tkColon - else: p.tok.tokType == tkOpr and identEq(p.tok.ident, "->") + else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" if hasRet and p.tok.indent < 0: getTok(p) optInd(p, result) @@ -2023,7 +2025,8 @@ proc parseTopLevelStmt(p: var TParser): PNode = if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break -proc parseString*(s: string; filename: string = ""; line: int = 0; +proc parseString*(s: string; cache: IdentCache; filename: string = ""; + line: int = 0; errorHandler: TErrorHandler = nil): PNode = ## Parses a string into an AST, returning the top node. ## `filename` and `line`, although optional, provide info so that the @@ -2036,7 +2039,7 @@ proc parseString*(s: string; filename: string = ""; line: int = 0; # XXX for now the builtin 'parseStmt/Expr' functions do not know about strong # spaces... parser.lex.errorHandler = errorHandler - openParser(parser, filename, stream, false) + openParser(parser, filename, stream, cache, false) result = parser.parseAll closeParser(parser) diff --git a/compiler/passaux.nim b/compiler/passaux.nim index d4361a671..eeaf12953 100644 --- a/compiler/passaux.nim +++ b/compiler/passaux.nim @@ -10,9 +10,11 @@ ## implements some little helper passes import - strutils, ast, astalgo, passes, msgs, options, idgen + strutils, ast, astalgo, passes, idents, msgs, options, idgen -proc verboseOpen(s: PSym): PPassContext = +from modulegraphs import ModuleGraph + +proc verboseOpen(graph: ModuleGraph; s: PSym; cache: IdentCache): PPassContext = #MessageOut('compiling ' + s.name.s); result = nil # we don't need a context rawMessage(hintProcessing, s.name.s) diff --git a/compiler/passes.nim b/compiler/passes.nim index b7642e3e4..3cc15147e 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -13,7 +13,7 @@ import strutils, lists, options, ast, astalgo, llstream, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, magicsys, nversion, - nimsets, syntaxes, times, rodread, idgen + nimsets, syntaxes, times, rodread, idgen, modulegraphs type TPassContext* = object of RootObj # the pass's context @@ -21,9 +21,9 @@ type PPassContext* = ref TPassContext - TPassOpen* = proc (module: PSym): PPassContext {.nimcall.} + TPassOpen* = proc (graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext {.nimcall.} TPassOpenCached* = - proc (module: PSym, rd: PRodReader): PPassContext {.nimcall.} + proc (graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext {.nimcall.} TPassClose* = proc (p: PPassContext, n: PNode): PNode {.nimcall.} TPassProcess* = proc (p: PPassContext, topLevelStmt: PNode): PNode {.nimcall.} @@ -48,8 +48,8 @@ proc makePass*(open: TPassOpen = nil, # the semantic checker needs these: var - gImportModule*: proc (m: PSym, fileIdx: int32): PSym {.nimcall.} - gIncludeFile*: proc (m: PSym, fileIdx: int32): PNode {.nimcall.} + gImportModule*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PSym {.nimcall.} + gIncludeFile*: proc (graph: ModuleGraph; m: PSym, fileIdx: int32; cache: IdentCache): PNode {.nimcall.} # implementation @@ -90,28 +90,32 @@ proc registerPass*(p: TPass) = gPasses[gPassesLen] = p inc(gPassesLen) -proc carryPass*(p: TPass, module: PSym, m: TPassData): TPassData = - var c = p.open(module) +proc carryPass*(g: ModuleGraph; p: TPass, module: PSym; cache: IdentCache; + m: TPassData): TPassData = + var c = p.open(g, module, cache) result.input = p.process(c, m.input) result.closeOutput = if p.close != nil: p.close(c, m.closeOutput) else: m.closeOutput -proc carryPasses*(nodes: PNode, module: PSym, passes: TPasses) = +proc carryPasses*(g: ModuleGraph; nodes: PNode, module: PSym; + cache: IdentCache; passes: TPasses) = var passdata: TPassData passdata.input = nodes for pass in passes: - passdata = carryPass(pass, module, passdata) + passdata = carryPass(g, pass, module, cache, passdata) -proc openPasses(a: var TPassContextArray, module: PSym) = +proc openPasses(g: ModuleGraph; a: var TPassContextArray; + module: PSym; cache: IdentCache) = for i in countup(0, gPassesLen - 1): if not isNil(gPasses[i].open): - a[i] = gPasses[i].open(module) + a[i] = gPasses[i].open(g, module, cache) else: a[i] = nil -proc openPassesCached(a: var TPassContextArray, module: PSym, rd: PRodReader) = +proc openPassesCached(g: ModuleGraph; a: var TPassContextArray, module: PSym, + rd: PRodReader) = for i in countup(0, gPassesLen - 1): if not isNil(gPasses[i].openCached): - a[i] = gPasses[i].openCached(module, rd) + a[i] = gPasses[i].openCached(g, module, rd) if a[i] != nil: a[i].fromCache = true else: @@ -145,24 +149,35 @@ proc closePassesCached(a: var TPassContextArray) = m = gPasses[i].close(a[i], m) a[i] = nil # free the memory here +proc resolveMod(module, relativeTo: string): int32 = + let fullPath = findModule(module, relativeTo) + if fullPath.len == 0: + result = InvalidFileIDX + else: + result = fullPath.fileInfoIdx + proc processImplicits(implicits: seq[string], nodeKind: TNodeKind, - a: var TPassContextArray) = + a: var TPassContextArray; m: PSym) = + # XXX fixme this should actually be relative to the config file! + let relativeTo = m.info.toFullPath for module in items(implicits): - var importStmt = newNodeI(nodeKind, gCmdLineInfo) - var str = newStrNode(nkStrLit, module) - str.info = gCmdLineInfo - importStmt.addSon str - if not processTopLevelStmt(importStmt, a): break - -proc processModule*(module: PSym, stream: PLLStream, - rd: PRodReader): bool {.discardable.} = + # implicit imports should not lead to a module importing itself + if m.position != resolveMod(module, relativeTo): + var importStmt = newNodeI(nodeKind, gCmdLineInfo) + var str = newStrNode(nkStrLit, module) + str.info = gCmdLineInfo + importStmt.addSon str + if not processTopLevelStmt(importStmt, a): break + +proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream, + rd: PRodReader; cache: IdentCache): bool {.discardable.} = var p: TParsers a: TPassContextArray s: PLLStream fileIdx = module.fileIdx if rd == nil: - openPasses(a, module) + openPasses(graph, a, module, cache) if stream == nil: let filename = fileIdx.toFullPathConsiderDirty s = llStreamOpen(filename, fmRead) @@ -172,15 +187,15 @@ proc processModule*(module: PSym, stream: PLLStream, else: s = stream while true: - openParsers(p, fileIdx, s) + openParsers(p, fileIdx, s, cache) if sfSystemModule notin module.flags: # XXX what about caching? no processing then? what if I change the # modules to include between compilation runs? we'd need to track that # in ROD files. I think we should enable this feature only # for the interactive mode. - processImplicits implicitImports, nkImportStmt, a - processImplicits implicitIncludes, nkIncludeStmt, a + processImplicits implicitImports, nkImportStmt, a, module + processImplicits implicitIncludes, nkIncludeStmt, a, module while true: var n = parseTopLevelStmt(p) @@ -202,7 +217,7 @@ proc processModule*(module: PSym, stream: PLLStream, # id synchronization point for more consistent code generation: idSynchronizationPoint(1000) else: - openPassesCached(a, module, rd) + openPassesCached(graph, a, module, rd) var n = loadInitSection(rd) for i in countup(0, sonsLen(n) - 1): processTopLevelStmtCached(n.sons[i], a) closePassesCached(a) diff --git a/compiler/pbraces.nim b/compiler/pbraces.nim index 00f83a11e..df9204be6 100644 --- a/compiler/pbraces.nim +++ b/compiler/pbraces.nim @@ -1,18 +1,1780 @@ # # # The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # +# This module implements the parser of the braces Nim syntax. + import - llstream, lexer, parser, idents, strutils, ast, msgs + llstream, lexer, idents, strutils, ast, astalgo, msgs + +from parser import TParser + +proc getTok(p: var TParser) = + ## Get the next token from the parser's lexer, and store it in the parser's + ## `tok` member. + rawGetTok(p.lex, p.tok) + +proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream; + cache: IdentCache) = + ## Open a parser, using the given arguments to set up its internal state. + ## + initToken(p.tok) + openLexer(p.lex, fileIdx, inputStream, cache) + getTok(p) # read the first token + p.lex.allowTabs = true + +proc openParser*(p: var TParser, filename: string, inputStream: PLLStream; + cache: IdentCache) = + openParser(p, filename.fileInfoIdx, inputStream, cache) + +proc closeParser*(p: var TParser) = + ## Close a parser, freeing up its resources. + closeLexer(p.lex) + +proc parMessage(p: TParser, msg: TMsgKind, arg = "") = + ## Produce and emit the parser message `arg` to output. + lexMessageTok(p.lex, msg, p.tok, arg) + +proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) = + ## Produce and emit a parser message to output about the token `tok` + parMessage(p, msg, prettyTok(tok)) + +proc rawSkipComment(p: var TParser, node: PNode) = + if p.tok.tokType == tkComment: + if node != nil: + if node.comment == nil: node.comment = "" + add(node.comment, p.tok.literal) + else: + parMessage(p, errInternal, "skipComment") + getTok(p) + +proc skipComment(p: var TParser, node: PNode) = + rawSkipComment(p, node) + +proc flexComment(p: var TParser, node: PNode) = + rawSkipComment(p, node) + +proc skipInd(p: var TParser) = discard +proc optPar(p: var TParser) = discard + +proc optInd(p: var TParser, n: PNode) = + skipComment(p, n) + +proc getTokNoInd(p: var TParser) = + getTok(p) + +proc expectIdentOrKeyw(p: TParser) = + if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType): + lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) + +proc expectIdent(p: TParser) = + if p.tok.tokType != tkSymbol: + lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) + +proc eat(p: var TParser, tokType: TTokType) = + ## Move the parser to the next token if the current token is of type + ## `tokType`, otherwise error. + if p.tok.tokType == tokType: + getTok(p) + else: + lexMessageTok(p.lex, errTokenExpected, p.tok, TokTypeToStr[tokType]) + +proc parLineInfo(p: TParser): TLineInfo = + ## Retrieve the line information associated with the parser's current state. + result = getLineInfo(p.lex, p.tok) + +proc indAndComment(p: var TParser, n: PNode) = + rawSkipComment(p, n) + +proc newNodeP(kind: TNodeKind, p: TParser): PNode = + result = newNodeI(kind, parLineInfo(p)) + +proc newIntNodeP(kind: TNodeKind, intVal: BiggestInt, p: TParser): PNode = + result = newNodeP(kind, p) + result.intVal = intVal + +proc newFloatNodeP(kind: TNodeKind, floatVal: BiggestFloat, + p: TParser): PNode = + result = newNodeP(kind, p) + result.floatVal = floatVal + +proc newStrNodeP(kind: TNodeKind, strVal: string, p: TParser): PNode = + result = newNodeP(kind, p) + result.strVal = strVal + +proc newIdentNodeP(ident: PIdent, p: TParser): PNode = + result = newNodeP(nkIdent, p) + result.ident = ident + +proc parseExpr(p: var TParser): PNode +proc parseStmt(p: var TParser): PNode +proc parseTypeDesc(p: var TParser): PNode +proc parseDoBlocks(p: var TParser, call: PNode) +proc parseParamList(p: var TParser, retColon = true): PNode +proc parseStmtPragma(p: var TParser): PNode +proc parseCase(p: var TParser): PNode +proc parseTry(p: var TParser): PNode + +proc isSigilLike(tok: TToken): bool {.inline.} = + result = tok.tokType == tkOpr and tok.ident.s[0] == '@' + +proc isAt(tok: TToken): bool {.inline.} = + tok.tokType == tkOpr and tok.ident.s == "@" and tok.strongSpaceB == 0 + +proc isRightAssociative(tok: TToken): bool {.inline.} = + ## Determines whether the token is right assocative. + result = tok.tokType == tkOpr and tok.ident.s[0] == '^' + # or (let L = tok.ident.s.len; L > 1 and tok.ident.s[L-1] == '>')) + +proc getPrecedence(tok: TToken): int = + ## Calculates the precedence of the given token. + template considerStrongSpaces(x): untyped = x + + case tok.tokType + of tkOpr: + let L = tok.ident.s.len + let relevantChar = tok.ident.s[0] + + # arrow like? + if L > 1 and tok.ident.s[L-1] == '>' and + tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1) + + template considerAsgn(value: untyped) = + result = if tok.ident.s[L-1] == '=': 1 else: value + + case relevantChar + of '$', '^': considerAsgn(10) + of '*', '%', '/', '\\': considerAsgn(9) + of '~': result = 8 + of '+', '-', '|': considerAsgn(8) + of '&': considerAsgn(7) + of '=', '<', '>', '!': result = 5 + of '.': considerAsgn(6) + of '?': result = 2 + else: considerAsgn(2) + of tkDiv, tkMod, tkShl, tkShr: result = 9 + of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5 + of tkDotDot: result = 6 + of tkAnd: result = 4 + of tkOr, tkXor, tkPtr, tkRef: result = 3 + else: return -10 + result = considerStrongSpaces(result) + +proc isOperator(tok: TToken): bool = + ## Determines if the given token is an operator type token. + tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs, + tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor} + +proc isUnary(p: TParser): bool = + ## Check if the current parser token is a unary operator + if p.tok.tokType in {tkOpr, tkDotDot}: + result = true + +proc checkBinary(p: TParser) {.inline.} = + ## Check if the current parser token is a binary operator. + # we don't check '..' here as that's too annoying + discard + +#| module = stmt ^* (';' / IND{=}) +#| +#| comma = ',' COMMENT? +#| semicolon = ';' COMMENT? +#| colon = ':' COMMENT? +#| colcom = ':' COMMENT? +#| +#| operator = OP0 | OP1 | OP2 | OP3 | OP4 | OP5 | OP6 | OP7 | OP8 | OP9 +#| | 'or' | 'xor' | 'and' +#| | 'is' | 'isnot' | 'in' | 'notin' | 'of' +#| | 'div' | 'mod' | 'shl' | 'shr' | 'not' | 'static' | '..' +#| +#| prefixOperator = operator +#| +#| optInd = COMMENT? +#| optPar = (IND{>} | IND{=})? +#| +#| simpleExpr = arrowExpr (OP0 optInd arrowExpr)* +#| arrowExpr = assignExpr (OP1 optInd assignExpr)* +#| assignExpr = orExpr (OP2 optInd orExpr)* +#| orExpr = andExpr (OP3 optInd andExpr)* +#| andExpr = cmpExpr (OP4 optInd cmpExpr)* +#| cmpExpr = sliceExpr (OP5 optInd sliceExpr)* +#| sliceExpr = ampExpr (OP6 optInd ampExpr)* +#| ampExpr = plusExpr (OP7 optInd plusExpr)* +#| plusExpr = mulExpr (OP8 optInd mulExpr)* +#| mulExpr = dollarExpr (OP9 optInd dollarExpr)* +#| dollarExpr = primary (OP10 optInd primary)* + +proc colcom(p: var TParser, n: PNode) = + skipComment(p, n) + +proc parseSymbol(p: var TParser, allowNil = false): PNode = + #| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`' + #| | IDENT | 'addr' | 'type' + case p.tok.tokType + of tkSymbol, tkAddr, tkType: + result = newIdentNodeP(p.tok.ident, p) + getTok(p) + of tkAccent: + result = newNodeP(nkAccQuoted, p) + getTok(p) + while true: + case p.tok.tokType + of tkAccent: + if result.len == 0: + parMessage(p, errIdentifierExpected, p.tok) + break + of tkOpr, tkDot, tkDotDot, tkEquals, tkParLe..tkParDotRi: + var accm = "" + while p.tok.tokType in {tkOpr, tkDot, tkDotDot, tkEquals, + tkParLe..tkParDotRi}: + accm.add(tokToStr(p.tok)) + getTok(p) + result.add(newIdentNodeP(p.lex.cache.getIdent(accm), p)) + of tokKeywordLow..tokKeywordHigh, tkSymbol, tkIntLit..tkCharLit: + result.add(newIdentNodeP(p.lex.cache.getIdent(tokToStr(p.tok)), p)) + getTok(p) + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkAccent) + else: + if allowNil and p.tok.tokType == tkNil: + result = newNodeP(nkNilLit, p) + getTok(p) + else: + parMessage(p, errIdentifierExpected, p.tok) + # BUGFIX: We must consume a token here to prevent endless loops! + # But: this really sucks for idetools and keywords, so we don't do it + # if it is a keyword: + if not isKeyword(p.tok.tokType): getTok(p) + result = ast.emptyNode + +proc colonOrEquals(p: var TParser, a: PNode): PNode = + if p.tok.tokType == tkColon: + result = newNodeP(nkExprColonExpr, p) + getTok(p) + #optInd(p, result) + addSon(result, a) + addSon(result, parseExpr(p)) + elif p.tok.tokType == tkEquals: + result = newNodeP(nkExprEqExpr, p) + getTok(p) + #optInd(p, result) + addSon(result, a) + addSon(result, parseExpr(p)) + else: + result = a + +proc exprColonEqExpr(p: var TParser): PNode = + #| exprColonEqExpr = expr (':'|'=' expr)? + var a = parseExpr(p) + result = colonOrEquals(p, a) + +proc exprList(p: var TParser, endTok: TTokType, result: PNode) = + #| exprList = expr ^+ comma + getTok(p) + optInd(p, result) + while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): + var a = parseExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + +proc dotExpr(p: var TParser, a: PNode): PNode = + #| dotExpr = expr '.' optInd symbol + var info = p.parLineInfo + getTok(p) + result = newNodeI(nkDotExpr, info) + optInd(p, result) + addSon(result, a) + addSon(result, parseSymbol(p)) + +proc qualifiedIdent(p: var TParser): PNode = + #| qualifiedIdent = symbol ('.' optInd symbol)? + result = parseSymbol(p) + if p.tok.tokType == tkDot: result = dotExpr(p, result) + +proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) = + assert(endTok in {tkCurlyLe, tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) + getTok(p) + optInd(p, result) + while p.tok.tokType != endTok and p.tok.tokType != tkEof: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, endTok) + +proc exprColonEqExprList(p: var TParser, kind: TNodeKind, + endTok: TTokType): PNode = + #| exprColonEqExprList = exprColonEqExpr (comma exprColonEqExpr)* (comma)? + result = newNodeP(kind, p) + exprColonEqExprListAux(p, endTok, result) + +proc setOrTableConstr(p: var TParser): PNode = + result = newNodeP(nkCurly, p) + getTok(p) + optInd(p, result) + if p.tok.tokType == tkColon: + getTok(p) # skip ':' + result.kind = nkTableConstr + else: + while p.tok.tokType notin {tkBracketDotRi, tkEof}: + var a = exprColonEqExpr(p) + if a.kind == nkExprColonExpr: result.kind = nkTableConstr + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkBracketDotRi) + +proc parseCast(p: var TParser): PNode = + #| castExpr = 'cast' '[' optInd typeDesc optPar ']' '(' optInd expr optPar ')' + result = newNodeP(nkCast, p) + getTok(p) + eat(p, tkBracketLe) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + optPar(p) + eat(p, tkBracketRi) + eat(p, tkParLe) + optInd(p, result) + addSon(result, parseExpr(p)) + optPar(p) + eat(p, tkParRi) + +proc setBaseFlags(n: PNode, base: TNumericalBase) = + case base + of base10: discard + of base2: incl(n.flags, nfBase2) + of base8: incl(n.flags, nfBase8) + of base16: incl(n.flags, nfBase16) + +proc parseGStrLit(p: var TParser, a: PNode): PNode = + case p.tok.tokType + of tkGStrLit: + result = newNodeP(nkCallStrLit, p) + addSon(result, a) + addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) + getTok(p) + of tkGTripleStrLit: + result = newNodeP(nkCallStrLit, p) + addSon(result, a) + addSon(result, newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + getTok(p) + else: + result = a + +type + TPrimaryMode = enum pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix + +proc complexOrSimpleStmt(p: var TParser): PNode +proc simpleExpr(p: var TParser, mode = pmNormal): PNode + +proc semiStmtList(p: var TParser, result: PNode) = + inc p.inSemiStmtList + result.add(complexOrSimpleStmt(p)) + while p.tok.tokType == tkSemiColon: + getTok(p) + optInd(p, result) + result.add(complexOrSimpleStmt(p)) + dec p.inSemiStmtList + result.kind = nkStmtListExpr + +proc parsePar(p: var TParser): PNode = + #| parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try' + #| | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let' + #| | 'when' | 'var' | 'mixin' + #| par = '(' optInd + #| ( &parKeyw complexOrSimpleStmt ^+ ';' + #| | ';' complexOrSimpleStmt ^+ ';' + #| | pragmaStmt + #| | simpleExpr ( ('=' expr (';' complexOrSimpleStmt ^+ ';' )? ) + #| | (':' expr (',' exprColonEqExpr ^+ ',' )? ) ) ) + #| optPar ')' + # + # unfortunately it's ambiguous: (expr: expr) vs (exprStmt); however a + # leading ';' could be used to enforce a 'stmt' context ... + result = newNodeP(nkPar, p) + getTok(p) + optInd(p, result) + if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase, + tkTry, tkDefer, tkFinally, tkExcept, tkFor, tkBlock, + tkConst, tkLet, tkWhen, tkVar, + tkMixin}: + # XXX 'bind' used to be an expression, so we exclude it here; + # tests/reject/tbind2 fails otherwise. + semiStmtList(p, result) + elif p.tok.tokType == tkSemiColon: + # '(;' enforces 'stmt' context: + getTok(p) + optInd(p, result) + semiStmtList(p, result) + elif p.tok.tokType == tkCurlyDotLe: + result.add(parseStmtPragma(p)) + elif p.tok.tokType != tkParRi: + var a = simpleExpr(p) + if p.tok.tokType == tkEquals: + # special case: allow assignments + getTok(p) + optInd(p, result) + let b = parseExpr(p) + let asgn = newNodeI(nkAsgn, a.info, 2) + asgn.sons[0] = a + asgn.sons[1] = b + result.add(asgn) + if p.tok.tokType == tkSemiColon: + semiStmtList(p, result) + elif p.tok.tokType == tkSemiColon: + # stmt context: + result.add(a) + semiStmtList(p, result) + else: + a = colonOrEquals(p, a) + result.add(a) + if p.tok.tokType == tkComma: + getTok(p) + skipComment(p, a) + while p.tok.tokType != tkParRi and p.tok.tokType != tkEof: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkParRi) + +proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode = + #| literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT + #| | UINT_LIT | UINT8_LIT | UINT16_LIT | UINT32_LIT | UINT64_LIT + #| | FLOAT_LIT | FLOAT32_LIT | FLOAT64_LIT + #| | STR_LIT | RSTR_LIT | TRIPLESTR_LIT + #| | CHAR_LIT + #| | NIL + #| generalizedLit = GENERALIZED_STR_LIT | GENERALIZED_TRIPLESTR_LIT + #| identOrLiteral = generalizedLit | symbol | literal + #| | par | arrayConstr | setOrTableConstr + #| | castExpr + #| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' + #| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' + case p.tok.tokType + of tkSymbol, tkType, tkAddr: + result = newIdentNodeP(p.tok.ident, p) + getTok(p) + result = parseGStrLit(p, result) + of tkAccent: + result = parseSymbol(p) # literals + of tkIntLit: + result = newIntNodeP(nkIntLit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt8Lit: + result = newIntNodeP(nkInt8Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt16Lit: + result = newIntNodeP(nkInt16Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt32Lit: + result = newIntNodeP(nkInt32Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkInt64Lit: + result = newIntNodeP(nkInt64Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUIntLit: + result = newIntNodeP(nkUIntLit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt8Lit: + result = newIntNodeP(nkUInt8Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt16Lit: + result = newIntNodeP(nkUInt16Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt32Lit: + result = newIntNodeP(nkUInt32Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkUInt64Lit: + result = newIntNodeP(nkUInt64Lit, p.tok.iNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloatLit: + result = newFloatNodeP(nkFloatLit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat32Lit: + result = newFloatNodeP(nkFloat32Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat64Lit: + result = newFloatNodeP(nkFloat64Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkFloat128Lit: + result = newFloatNodeP(nkFloat128Lit, p.tok.fNumber, p) + setBaseFlags(result, p.tok.base) + getTok(p) + of tkStrLit: + result = newStrNodeP(nkStrLit, p.tok.literal, p) + getTok(p) + of tkRStrLit: + result = newStrNodeP(nkRStrLit, p.tok.literal, p) + getTok(p) + of tkTripleStrLit: + result = newStrNodeP(nkTripleStrLit, p.tok.literal, p) + getTok(p) + of tkCharLit: + result = newIntNodeP(nkCharLit, ord(p.tok.literal[0]), p) + getTok(p) + of tkNil: + result = newNodeP(nkNilLit, p) + getTok(p) + of tkParLe: + # () constructor + if mode in {pmTypeDesc, pmTypeDef}: + result = exprColonEqExprList(p, nkPar, tkParRi) + else: + result = parsePar(p) + of tkBracketDotLe: + # {} constructor + result = setOrTableConstr(p) + of tkBracketLe: + # [] constructor + result = exprColonEqExprList(p, nkBracket, tkBracketRi) + of tkCast: + result = parseCast(p) + else: + parMessage(p, errExprExpected, p.tok) + getTok(p) # we must consume a token here to prevend endless loops! + result = ast.emptyNode + +proc namedParams(p: var TParser, callee: PNode, + kind: TNodeKind, endTok: TTokType): PNode = + let a = callee + result = newNodeP(kind, p) + addSon(result, a) + exprColonEqExprListAux(p, endTok, result) + +proc parseMacroColon(p: var TParser, x: PNode): PNode +proc primarySuffix(p: var TParser, r: PNode): PNode = + #| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? + #| | doBlocks + #| | '.' optInd symbol generalizedLit? + #| | '[' optInd indexExprList optPar ']' + #| | '{' optInd indexExprList optPar '}' + #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax + result = r + + template somePar() = discard + while p.tok.indent < 0: + case p.tok.tokType + of tkParLe: + somePar() + result = namedParams(p, result, nkCall, tkParRi) + if result.len > 1 and result.sons[1].kind == nkExprColonExpr: + result.kind = nkObjConstr + else: + parseDoBlocks(p, result) + of tkDo: + var a = result + result = newNodeP(nkCall, p) + addSon(result, a) + parseDoBlocks(p, result) + of tkDot: + result = dotExpr(p, result) + result = parseGStrLit(p, result) + of tkBracketLe: + somePar() + result = namedParams(p, result, nkBracketExpr, tkBracketRi) + of tkBracketDotLe: + somePar() + result = namedParams(p, result, nkCurlyExpr, tkBracketDotRi) + of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkAddr, tkType: + if p.inPragma == 0: + # actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet + # solution, but pragmas.nim can't handle that + let a = result + result = newNodeP(nkCommand, p) + addSon(result, a) + when true: + addSon result, parseExpr(p) + else: + while p.tok.tokType != tkEof: + let x = parseExpr(p) + addSon(result, x) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, x) + if p.tok.tokType == tkDo: + parseDoBlocks(p, result) + else: + result = parseMacroColon(p, result) + break + else: + break + +proc primary(p: var TParser, mode: TPrimaryMode): PNode +proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode + +proc parseOperators(p: var TParser, headNode: PNode, + limit: int, mode: TPrimaryMode): PNode = + result = headNode + # expand while operators have priorities higher than 'limit' + var opPrec = getPrecedence(p.tok) + let modeB = if mode == pmTypeDef: pmTypeDesc else: mode + # the operator itself must not start on a new line: + while opPrec >= limit and p.tok.indent < 0 and not isAt(p.tok): + checkBinary(p) + var leftAssoc = 1-ord(isRightAssociative(p.tok)) + var a = newNodeP(nkInfix, p) + var opNode = newIdentNodeP(p.tok.ident, p) # skip operator: + getTok(p) + optInd(p, a) + # read sub-expression with higher priority: + var b = simpleExprAux(p, opPrec + leftAssoc, modeB) + addSon(a, opNode) + addSon(a, result) + addSon(a, b) + result = a + opPrec = getPrecedence(p.tok) + +proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode = + result = primary(p, mode) + result = parseOperators(p, result, limit, mode) + +proc simpleExpr(p: var TParser, mode = pmNormal): PNode = + result = simpleExprAux(p, -1, mode) + +proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode = + #| condExpr = expr colcom expr optInd + #| ('elif' expr colcom expr optInd)* + #| 'else' colcom expr + #| ifExpr = 'if' condExpr + #| whenExpr = 'when' condExpr + result = newNodeP(kind, p) + while true: + getTok(p) # skip `if`, `elif` + var branch = newNodeP(nkElifExpr, p) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseExpr(p)) + optInd(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + var branch = newNodeP(nkElseExpr, p) + eat(p, tkElse) + colcom(p, branch) + addSon(branch, parseExpr(p)) + addSon(result, branch) + +proc parsePragma(p: var TParser): PNode = + result = newNodeP(nkPragma, p) + inc p.inPragma + if isAt(p.tok): + while isAt(p.tok): + getTok(p) + var a = parseExpr(p) + optInd(p, a) + if a.kind in nkCallKinds and a.len == 2: + let repaired = newNodeI(nkExprColonExpr, a.info) + repaired.add a[0] + repaired.add a[1] + a = repaired + addSon(result, a) + skipComment(p, a) + else: + getTok(p) + optInd(p, result) + while p.tok.tokType notin {tkCurlyDotRi, tkCurlyRi, tkEof}: + var a = exprColonEqExpr(p) + addSon(result, a) + if p.tok.tokType == tkComma: + getTok(p) + skipComment(p, a) + optPar(p) + if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: getTok(p) + else: parMessage(p, errTokenExpected, ".}") + dec p.inPragma + +proc identVis(p: var TParser; allowDot=false): PNode = + #| identVis = symbol opr? # postfix position + #| identVisDot = symbol '.' optInd symbol opr? + var a = parseSymbol(p) + if p.tok.tokType == tkOpr: + result = newNodeP(nkPostfix, p) + addSon(result, newIdentNodeP(p.tok.ident, p)) + addSon(result, a) + getTok(p) + elif p.tok.tokType == tkDot and allowDot: + result = dotExpr(p, a) + else: + result = a + +proc identWithPragma(p: var TParser; allowDot=false): PNode = + #| identWithPragma = identVis pragma? + #| identWithPragmaDot = identVisDot pragma? + var a = identVis(p, allowDot) + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): + result = newNodeP(nkPragmaExpr, p) + addSon(result, a) + addSon(result, parsePragma(p)) + else: + result = a + +type + TDeclaredIdentFlag = enum + withPragma, # identifier may have pragma + withBothOptional # both ':' and '=' parts are optional + TDeclaredIdentFlags = set[TDeclaredIdentFlag] + +proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = + #| declColonEquals = identWithPragma (comma identWithPragma)* comma? + #| (':' optInd typeDesc)? ('=' optInd expr)? + #| identColonEquals = ident (comma ident)* comma? + #| (':' optInd typeDesc)? ('=' optInd expr)?) + var a: PNode + result = newNodeP(nkIdentDefs, p) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + if withPragma in flags: a = identWithPragma(p) + else: a = parseSymbol(p) + if a.kind == nkEmpty: return + else: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType != tkEquals and withBothOptional notin flags: + parMessage(p, errColonOrEqualsExpected, p.tok) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + +proc parseTuple(p: var TParser): PNode = + result = newNodeP(nkTupleTy, p) + getTok(p) + if p.tok.tokType in {tkBracketLe, tkCurlyLe}: + let usedCurly = p.tok.tokType == tkCurlyLe + getTok(p) + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = parseIdentColonEquals(p, {}) + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + if usedCurly: eat(p, tkCurlyRi) + else: eat(p, tkBracketRi) + else: + result = newNodeP(nkTupleClassTy, p) + +proc parseParamList(p: var TParser, retColon = true): PNode = + #| paramList = '(' declColonEquals ^* (comma/semicolon) ')' + #| paramListArrow = paramList? ('->' optInd typeDesc)? + #| paramListColon = paramList? (':' optInd typeDesc)? + var a: PNode + result = newNodeP(nkFormalParams, p) + addSon(result, ast.emptyNode) # return type + let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0 + if hasParLe: + getTok(p) + optInd(p, result) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + a = parseIdentColonEquals(p, {withBothOptional, withPragma}) + of tkParRi: + break + else: + parMessage(p, errTokenExpected, ")") + break + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkParRi) + let hasRet = if retColon: p.tok.tokType == tkColon + else: p.tok.tokType == tkOpr and p.tok.ident.s == "->" + if hasRet and p.tok.indent < 0: + getTok(p) + optInd(p, result) + result.sons[0] = parseTypeDesc(p) + elif not retColon and not hasParle: + # Mark as "not there" in order to mark for deprecation in the semantic pass: + result = ast.emptyNode + +proc optPragmas(p: var TParser): PNode = + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): + result = parsePragma(p) + else: + result = ast.emptyNode + +proc parseDoBlock(p: var TParser): PNode = + #| doBlock = 'do' paramListArrow pragmas? colcom stmt + let info = parLineInfo(p) + getTok(p) + let params = parseParamList(p, retColon=false) + let pragmas = optPragmas(p) + colcom(p, result) + result = newProcNode(nkDo, info, parseStmt(p), + params = params, + pragmas = pragmas) + +proc parseDoBlocks(p: var TParser, call: PNode) = + #| doBlocks = doBlock ^* IND{=} + while p.tok.tokType == tkDo: + addSon(call, parseDoBlock(p)) + +proc parseCurlyStmt(p: var TParser): PNode = + result = newNodeP(nkStmtList, p) + eat(p, tkCurlyLe) + result.add parseStmt(p) + while p.tok.tokType notin {tkEof, tkCurlyRi}: + if p.tok.tokType == tkSemicolon: getTok(p) + elif p.tok.indent < 0: break + result.add parseStmt(p) + eat(p, tkCurlyRi) + +proc parseProcExpr(p: var TParser, isExpr: bool): PNode = + #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? + # either a proc type or a anonymous proc + let info = parLineInfo(p) + getTok(p) + let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0 + let params = parseParamList(p) + let pragmas = optPragmas(p) + if p.tok.tokType == tkCurlyLe and isExpr: + result = newProcNode(nkLambda, info, parseCurlyStmt(p), + params = params, + pragmas = pragmas) + else: + result = newNodeI(nkProcTy, info) + if hasSignature: + addSon(result, params) + addSon(result, pragmas) + +proc isExprStart(p: TParser): bool = + case p.tok.tokType + of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, + tkProc, tkIterator, tkBind, tkAddr, + tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, + tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut: + result = true + else: result = false + +proc parseSymbolList(p: var TParser, result: PNode, allowNil = false) = + while true: + var s = parseSymbol(p, allowNil) + if s.kind == nkEmpty: break + addSon(result, s) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, s) + +proc parseTypeDescKAux(p: var TParser, kind: TNodeKind, + mode: TPrimaryMode): PNode = + #| distinct = 'distinct' optInd typeDesc + result = newNodeP(kind, p) + getTok(p) + optInd(p, result) + if not isOperator(p.tok) and isExprStart(p): + addSon(result, primary(p, mode)) + if kind == nkDistinctTy and p.tok.tokType in {tkWith, tkWithout}: + let nodeKind = if p.tok.tokType == tkWith: nkWith + else: nkWithout + getTok(p) + let list = newNodeP(nodeKind, p) + result.addSon list + parseSymbolList(p, list, allowNil = true) + +proc parseExpr(p: var TParser): PNode = + #| expr = (ifExpr + #| | whenExpr + #| | caseExpr + #| | tryExpr) + #| / simpleExpr + case p.tok.tokType: + of tkIf: result = parseIfExpr(p, nkIfExpr) + of tkWhen: result = parseIfExpr(p, nkWhenExpr) + of tkCase: result = parseCase(p) + of tkTry: result = parseTry(p) + else: result = simpleExpr(p) + +proc parseEnum(p: var TParser): PNode +proc parseObject(p: var TParser): PNode +proc parseTypeClass(p: var TParser): PNode + +proc primary(p: var TParser, mode: TPrimaryMode): PNode = + #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' + #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' + #| primary = typeKeyw typeDescK + #| / prefixOperator* identOrLiteral primarySuffix* + #| / 'static' primary + #| / 'bind' primary + if isOperator(p.tok): + let isSigil = isSigilLike(p.tok) + result = newNodeP(nkPrefix, p) + var a = newIdentNodeP(p.tok.ident, p) + addSon(result, a) + getTok(p) + optInd(p, a) + if isSigil: + #XXX prefix operators + addSon(result, primary(p, pmSkipSuffix)) + result = primarySuffix(p, result) + else: + addSon(result, primary(p, pmNormal)) + return + + case p.tok.tokType: + of tkTuple: result = parseTuple(p) + of tkProc: result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}) + of tkIterator: + result = parseProcExpr(p, mode notin {pmTypeDesc, pmTypeDef}) + if result.kind == nkLambda: result.kind = nkIteratorDef + else: result.kind = nkIteratorTy + of tkEnum: + if mode == pmTypeDef: + result = parseEnum(p) + else: + result = newNodeP(nkEnumTy, p) + getTok(p) + of tkObject: + if mode == pmTypeDef: + result = parseObject(p) + else: + result = newNodeP(nkObjectTy, p) + getTok(p) + of tkConcept: + if mode == pmTypeDef: + result = parseTypeClass(p) + else: + parMessage(p, errInvalidToken, p.tok) + of tkStatic: + let info = parLineInfo(p) + getTokNoInd(p) + let next = primary(p, pmNormal) + if next.kind == nkBracket and next.sonsLen == 1: + result = newNode(nkStaticTy, info, @[next.sons[0]]) + else: + result = newNode(nkStaticExpr, info, @[next]) + of tkBind: + result = newNodeP(nkBind, p) + getTok(p) + optInd(p, result) + addSon(result, primary(p, pmNormal)) + of tkVar: result = parseTypeDescKAux(p, nkVarTy, mode) + of tkOut: result = parseTypeDescKAux(p, nkVarTy, mode) + of tkRef: result = parseTypeDescKAux(p, nkRefTy, mode) + of tkPtr: result = parseTypeDescKAux(p, nkPtrTy, mode) + of tkDistinct: result = parseTypeDescKAux(p, nkDistinctTy, mode) + else: + result = identOrLiteral(p, mode) + if mode != pmSkipSuffix: + result = primarySuffix(p, result) + +proc parseTypeDesc(p: var TParser): PNode = + #| typeDesc = simpleExpr + result = simpleExpr(p, pmTypeDesc) + +proc parseTypeDefAux(p: var TParser): PNode = + #| typeDefAux = simpleExpr + #| | 'concept' typeClass + result = simpleExpr(p, pmTypeDef) + +proc makeCall(n: PNode): PNode = + ## Creates a call if the given node isn't already a call. + if n.kind in nkCallKinds: + result = n + else: + result = newNodeI(nkCall, n.info) + result.add n + +proc parseMacroColon(p: var TParser, x: PNode): PNode = + #| macroColon = ':' stmt? ( IND{=} 'of' exprList ':' stmt + #| | IND{=} 'elif' expr ':' stmt + #| | IND{=} 'except' exprList ':' stmt + #| | IND{=} 'else' ':' stmt )* + result = x + if p.tok.tokType == tkColon and p.tok.indent < 0: + result = makeCall(result) + getTok(p) + skipComment(p, result) + let stmtList = newNodeP(nkStmtList, p) + if p.tok.tokType notin {tkOf, tkElif, tkElse, tkExcept}: + let body = parseStmt(p) + stmtList.add body + #addSon(result, makeStmtList(body)) + while true: + var b: PNode + case p.tok.tokType + of tkOf: + b = newNodeP(nkOfBranch, p) + exprList(p, tkCurlyLe, b) + of tkElif: + b = newNodeP(nkElifBranch, p) + getTok(p) + optInd(p, b) + addSon(b, parseExpr(p)) + of tkExcept: + b = newNodeP(nkExceptBranch, p) + exprList(p, tkCurlyLe, b) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + addSon(b, parseCurlyStmt(p)) + addSon(stmtList, b) + if b.kind == nkElse: break + if stmtList.len == 1 and stmtList[0].kind == nkStmtList: + # to keep backwards compatibility (see tests/vm/tstringnil) + result.add stmtList[0] + else: + result.add stmtList + +proc parseExprStmt(p: var TParser): PNode = + #| exprStmt = simpleExpr + #| (( '=' optInd expr ) + #| / ( expr ^+ comma + #| doBlocks + #| / macroColon + #| ))? + var a = simpleExpr(p) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + var b = parseExpr(p) + result = newNodeI(nkAsgn, a.info) + addSon(result, a) + addSon(result, b) + else: + # simpleExpr parsed 'p a' from 'p a, b'? + if p.tok.indent < 0 and p.tok.tokType == tkComma and a.kind == nkCommand: + result = a + while true: + getTok(p) + optInd(p, result) + var e = parseExpr(p) + addSon(result, e) + if p.tok.tokType != tkComma: break + elif p.tok.indent < 0 and isExprStart(p): + if a.kind == nkCommand: + result = a + else: + result = newNode(nkCommand, a.info, @[a]) + while true: + var e = parseExpr(p) + addSon(result, e) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, result) + else: + result = a + if p.tok.tokType == tkDo and p.tok.indent < 0: + result = makeCall(result) + parseDoBlocks(p, result) + return result + result = parseMacroColon(p, result) + +proc parseModuleName(p: var TParser, kind: TNodeKind): PNode = + result = parseExpr(p) + +proc parseImport(p: var TParser, kind: TNodeKind): PNode = + #| importStmt = 'import' optInd expr + #| ((comma expr)* + #| / 'except' optInd (expr ^+ comma)) + result = newNodeP(kind, p) + getTok(p) # skip `import` or `export` + optInd(p, result) + var a = parseModuleName(p, kind) + addSon(result, a) + if p.tok.tokType in {tkComma, tkExcept}: + if p.tok.tokType == tkExcept: + result.kind = succ(kind) + getTok(p) + optInd(p, result) + while true: + # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseModuleName(p, kind) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseIncludeStmt(p: var TParser): PNode = + #| includeStmt = 'include' optInd expr ^+ comma + result = newNodeP(nkIncludeStmt, p) + getTok(p) # skip `import` or `include` + optInd(p, result) + while true: + # was: while p.tok.tokType notin {tkEof, tkSad, tkDed}: + var a = parseExpr(p) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseFromStmt(p: var TParser): PNode = + #| fromStmt = 'from' moduleName 'import' optInd expr (comma expr)* + result = newNodeP(nkFromStmt, p) + getTok(p) # skip `from` + optInd(p, result) + var a = parseModuleName(p, nkImportStmt) + addSon(result, a) #optInd(p, a); + eat(p, tkImport) + optInd(p, result) + while true: + # p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseExpr(p) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + #expectNl(p) + +proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode = + #| returnStmt = 'return' optInd expr? + #| raiseStmt = 'raise' optInd expr? + #| yieldStmt = 'yield' optInd expr? + #| discardStmt = 'discard' optInd expr? + #| breakStmt = 'break' optInd expr? + #| continueStmt = 'break' optInd expr? + result = newNodeP(kind, p) + getTok(p) + if p.tok.tokType == tkComment: + skipComment(p, result) + addSon(result, ast.emptyNode) + elif p.tok.indent >= 0 or not isExprStart(p): + # NL terminates: + addSon(result, ast.emptyNode) + else: + addSon(result, parseExpr(p)) + +proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode = + #| condStmt = expr colcom stmt COMMENT? + #| (IND{=} 'elif' expr colcom stmt)* + #| (IND{=} 'else' colcom stmt)? + #| ifStmt = 'if' condStmt + #| whenStmt = 'when' condStmt + result = newNodeP(kind, p) + while true: + getTok(p) # skip `if`, `when`, `elif` + var branch = newNodeP(nkElifBranch, p) + optInd(p, branch) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseCurlyStmt(p)) + skipComment(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + if p.tok.tokType == tkElse: + var branch = newNodeP(nkElse, p) + eat(p, tkElse) + addSon(branch, parseCurlyStmt(p)) + addSon(result, branch) + +proc parseWhile(p: var TParser): PNode = + #| whileStmt = 'while' expr colcom stmt + result = newNodeP(nkWhileStmt, p) + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseCase(p: var TParser): PNode = + #| ofBranch = 'of' exprList colcom stmt + #| ofBranches = ofBranch (IND{=} ofBranch)* + #| (IND{=} 'elif' expr colcom stmt)* + #| (IND{=} 'else' colcom stmt)? + #| caseStmt = 'case' expr ':'? COMMENT? + #| (IND{>} ofBranches DED + #| | IND{=} ofBranches) + var + b: PNode + inElif= false + result = newNodeP(nkCaseStmt, p) + getTok(p) + addSon(result, parseExpr(p)) + eat(p, tkCurlyLe) + skipComment(p, result) + + while true: + case p.tok.tokType + of tkOf: + if inElif: break + b = newNodeP(nkOfBranch, p) + exprList(p, tkCurlyLe, b) + of tkElif: + inElif = true + b = newNodeP(nkElifBranch, p) + getTok(p) + optInd(p, b) + addSon(b, parseExpr(p)) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + skipComment(p, b) + addSon(b, parseCurlyStmt(p)) + addSon(result, b) + if b.kind == nkElse: break + eat(p, tkCurlyRi) + +proc parseTry(p: var TParser): PNode = + #| tryStmt = 'try' colcom stmt &(IND{=}? 'except'|'finally') + #| (IND{=}? 'except' exprList colcom stmt)* + #| (IND{=}? 'finally' colcom stmt)? + #| tryExpr = 'try' colcom stmt &(optInd 'except'|'finally') + #| (optInd 'except' exprList colcom stmt)* + #| (optInd 'finally' colcom stmt)? + result = newNodeP(nkTryStmt, p) + getTok(p) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + var b: PNode = nil + while true: + case p.tok.tokType + of tkExcept: + b = newNodeP(nkExceptBranch, p) + exprList(p, tkCurlyLe, b) + of tkFinally: + b = newNodeP(nkFinally, p) + getTok(p) + else: break + skipComment(p, b) + addSon(b, parseCurlyStmt(p)) + addSon(result, b) + if b.kind == nkFinally: break + if b == nil: parMessage(p, errTokenExpected, "except") + +proc parseFor(p: var TParser): PNode = + #| forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt + result = newNodeP(nkForStmt, p) + getTokNoInd(p) + var a = identWithPragma(p) + addSon(result, a) + while p.tok.tokType == tkComma: + getTok(p) + optInd(p, a) + a = identWithPragma(p) + addSon(result, a) + eat(p, tkIn) + addSon(result, parseExpr(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseBlock(p: var TParser): PNode = + #| blockStmt = 'block' symbol? colcom stmt + result = newNodeP(nkBlockStmt, p) + getTokNoInd(p) + if p.tok.tokType == tkCurlyLe: addSon(result, ast.emptyNode) + else: addSon(result, parseSymbol(p)) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseStaticOrDefer(p: var TParser; k: TNodeKind): PNode = + #| staticStmt = 'static' colcom stmt + #| deferStmt = 'defer' colcom stmt + result = newNodeP(k, p) + getTok(p) + colcom(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseAsm(p: var TParser): PNode = + #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT) + result = newNodeP(nkAsmStmt, p) + getTokNoInd(p) + if p.tok.tokType == tkCurlyDotLe or isAt(p.tok): addSon(result, parsePragma(p)) + else: addSon(result, ast.emptyNode) + case p.tok.tokType + of tkStrLit: addSon(result, newStrNodeP(nkStrLit, p.tok.literal, p)) + of tkRStrLit: addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p)) + of tkTripleStrLit: addSon(result, + newStrNodeP(nkTripleStrLit, p.tok.literal, p)) + else: + parMessage(p, errStringLiteralExpected) + addSon(result, ast.emptyNode) + return + getTok(p) + +proc parseGenericParam(p: var TParser): PNode = + #| genericParam = symbol (comma symbol)* (colon expr)? ('=' optInd expr)? + var a: PNode + result = newNodeP(nkIdentDefs, p) + while true: + case p.tok.tokType + of tkSymbol, tkAccent: + a = parseSymbol(p) + if a.kind == nkEmpty: return + else: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseExpr(p)) + else: + addSon(result, ast.emptyNode) + +proc parseGenericParamList(p: var TParser): PNode = + #| genericParamList = '[' optInd + #| genericParam ^* (comma/semicolon) optPar ']' + result = newNodeP(nkGenericParams, p) + getTok(p) + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = parseGenericParam(p) + addSon(result, a) + if p.tok.tokType notin {tkComma, tkSemiColon}: break + getTok(p) + skipComment(p, a) + optPar(p) + eat(p, tkBracketRi) + +proc parsePattern(p: var TParser): PNode = + eat(p, tkBracketDotLe) + result = parseStmt(p) + eat(p, tkBracketDotRi) + +proc validInd(p: TParser): bool = p.tok.indent < 0 + +proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = + #| indAndComment = (IND{>} COMMENT)? | COMMENT? + #| routine = optInd identVis pattern? genericParamList? + #| paramListColon pragma? ('=' COMMENT? stmt)? indAndComment + result = newNodeP(kind, p) + getTok(p) + optInd(p, result) + addSon(result, identVis(p)) + if p.tok.tokType == tkBracketDotLe and p.validInd: + addSon(result, p.parsePattern) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkBracketLe and p.validInd: + result.add(p.parseGenericParamList) + else: + addSon(result, ast.emptyNode) + addSon(result, p.parseParamList) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, p.parsePragma) + else: + addSon(result, ast.emptyNode) + # empty exception tracking: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkCurlyLe: + addSon(result, parseCurlyStmt(p)) + else: + addSon(result, ast.emptyNode) + indAndComment(p, result) + +proc newCommentStmt(p: var TParser): PNode = + #| commentStmt = COMMENT + result = newNodeP(nkCommentStmt, p) + result.comment = p.tok.literal + getTok(p) + +type + TDefParser = proc (p: var TParser): PNode {.nimcall.} + +proc parseSection(p: var TParser, kind: TNodeKind, + defparser: TDefParser): PNode = + #| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED) + result = newNodeP(kind, p) + if kind != nkTypeSection: getTok(p) + skipComment(p, result) + if p.tok.tokType == tkParLe: + getTok(p) + skipComment(p, result) + while true: + case p.tok.tokType + of tkSymbol, tkAccent, tkParLe: + var a = defparser(p) + skipComment(p, a) + addSon(result, a) + of tkComment: + var a = newCommentStmt(p) + addSon(result, a) + of tkParRi: break + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkParRi) + if result.len == 0: parMessage(p, errIdentifierExpected, p.tok) + elif p.tok.tokType in {tkSymbol, tkAccent, tkBracketLe}: + # tkBracketLe is allowed for ``var [x, y] = ...`` tuple parsing + addSon(result, defparser(p)) + else: + parMessage(p, errIdentifierExpected, p.tok) + +proc parseConstant(p: var TParser): PNode = + #| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment + result = newNodeP(nkConstDef, p) + addSon(result, identWithPragma(p)) + if p.tok.tokType == tkColon: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDesc(p)) + else: + addSon(result, ast.emptyNode) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) + indAndComment(p, result) + +proc parseEnum(p: var TParser): PNode = + #| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+ + result = newNodeP(nkEnumTy, p) + getTok(p) + addSon(result, ast.emptyNode) + optInd(p, result) + flexComment(p, result) + eat(p, tkCurlyLe) + optInd(p, result) + while p.tok.tokType notin {tkEof, tkCurlyRi}: + var a = parseSymbol(p) + if a.kind == nkEmpty: return + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, a) + var b = a + a = newNodeP(nkEnumFieldDef, p) + addSon(a, b) + addSon(a, parseExpr(p)) + if p.tok.indent < 0: + rawSkipComment(p, a) + if p.tok.tokType == tkComma: + getTok(p) + rawSkipComment(p, a) + addSon(result, a) + eat(p, tkCurlyRi) + if result.len <= 1: + lexMessageTok(p.lex, errIdentifierExpected, p.tok, prettyTok(p.tok)) + +proc parseObjectPart(p: var TParser; needsCurly: bool): PNode +proc parseObjectWhen(p: var TParser): PNode = + result = newNodeP(nkRecWhen, p) + while true: + getTok(p) # skip `when`, `elif` + var branch = newNodeP(nkElifBranch, p) + optInd(p, branch) + addSon(branch, parseExpr(p)) + colcom(p, branch) + addSon(branch, parseObjectPart(p, true)) + flexComment(p, branch) + addSon(result, branch) + if p.tok.tokType != tkElif: break + if p.tok.tokType == tkElse: + var branch = newNodeP(nkElse, p) + eat(p, tkElse) + colcom(p, branch) + addSon(branch, parseObjectPart(p, true)) + flexComment(p, branch) + addSon(result, branch) + +proc parseObjectCase(p: var TParser): PNode = + result = newNodeP(nkRecCase, p) + getTokNoInd(p) + var a = newNodeP(nkIdentDefs, p) + addSon(a, identWithPragma(p)) + eat(p, tkColon) + addSon(a, parseTypeDesc(p)) + addSon(a, ast.emptyNode) + addSon(result, a) + eat(p, tkCurlyLe) + flexComment(p, result) + while true: + var b: PNode + case p.tok.tokType + of tkOf: + b = newNodeP(nkOfBranch, p) + exprList(p, tkColon, b) + of tkElse: + b = newNodeP(nkElse, p) + getTok(p) + else: break + colcom(p, b) + var fields = parseObjectPart(p, true) + if fields.kind == nkEmpty: + parMessage(p, errIdentifierExpected, p.tok) + fields = newNodeP(nkNilLit, p) # don't break further semantic checking + addSon(b, fields) + addSon(result, b) + if b.kind == nkElse: break + eat(p, tkCurlyRi) + +proc parseObjectPart(p: var TParser; needsCurly: bool): PNode = + if p.tok.tokType == tkCurlyLe: + result = newNodeP(nkRecList, p) + getTok(p) + rawSkipComment(p, result) + while true: + case p.tok.tokType + of tkCase, tkWhen, tkSymbol, tkAccent, tkNil, tkDiscard: + addSon(result, parseObjectPart(p, false)) + of tkCurlyRi: break + else: + parMessage(p, errIdentifierExpected, p.tok) + break + eat(p, tkCurlyRi) + else: + if needsCurly: + parMessage(p, errTokenExpected, "{") + case p.tok.tokType + of tkWhen: + result = parseObjectWhen(p) + of tkCase: + result = parseObjectCase(p) + of tkSymbol, tkAccent: + result = parseIdentColonEquals(p, {withPragma}) + if p.tok.indent < 0: rawSkipComment(p, result) + of tkNil, tkDiscard: + result = newNodeP(nkNilLit, p) + getTok(p) + else: + result = ast.emptyNode + +proc parseObject(p: var TParser): PNode = + result = newNodeP(nkObjectTy, p) + getTok(p) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, parsePragma(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkOf and p.tok.indent < 0: + var a = newNodeP(nkOfInherit, p) + getTok(p) + addSon(a, parseTypeDesc(p)) + addSon(result, a) + else: + addSon(result, ast.emptyNode) + skipComment(p, result) + # an initial IND{>} HAS to follow: + addSon(result, parseObjectPart(p, true)) + +proc parseTypeClassParam(p: var TParser): PNode = + if p.tok.tokType in {tkOut, tkVar}: + result = newNodeP(nkVarTy, p) + getTok(p) + result.addSon(p.parseSymbol) + else: + result = p.parseSymbol + +proc parseTypeClass(p: var TParser): PNode = + #| typeClassParam = ('var' | 'out')? symbol + #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? + #| &IND{>} stmt + result = newNodeP(nkTypeClassTy, p) + getTok(p) + var args = newNodeP(nkArgList, p) + addSon(result, args) + addSon(args, p.parseTypeClassParam) + while p.tok.tokType == tkComma: + getTok(p) + addSon(args, p.parseTypeClassParam) + if (p.tok.tokType == tkCurlyDotLe or isAt(p.tok)) and p.validInd: + addSon(result, parsePragma(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkOf and p.tok.indent < 0: + var a = newNodeP(nkOfInherit, p) + getTok(p) + while true: + addSon(a, parseTypeDesc(p)) + if p.tok.tokType != tkComma: break + getTok(p) + addSon(result, a) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkComment: + skipComment(p, result) + addSon(result, parseCurlyStmt(p)) + +proc parseTypeDef(p: var TParser): PNode = + #| + #| typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux + #| indAndComment? + result = newNodeP(nkTypeDef, p) + addSon(result, identWithPragma(p, allowDot=true)) + if p.tok.tokType == tkBracketLe and p.validInd: + addSon(result, parseGenericParamList(p)) + else: + addSon(result, ast.emptyNode) + if p.tok.tokType == tkEquals: + getTok(p) + optInd(p, result) + addSon(result, parseTypeDefAux(p)) + else: + addSon(result, ast.emptyNode) + indAndComment(p, result) # special extension! + +proc parseVarTuple(p: var TParser): PNode = + #| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr + result = newNodeP(nkVarTuple, p) + getTok(p) # skip '(' + optInd(p, result) + while p.tok.tokType in {tkSymbol, tkAccent}: + var a = identWithPragma(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + skipComment(p, a) + addSon(result, ast.emptyNode) # no type desc + optPar(p) + eat(p, tkBracketRi) + eat(p, tkEquals) + optInd(p, result) + addSon(result, parseExpr(p)) + +proc parseVariable(p: var TParser): PNode = + #| variable = (varTuple / identColonEquals) indAndComment + if p.tok.tokType == tkBracketLe: result = parseVarTuple(p) + else: result = parseIdentColonEquals(p, {withPragma}) + indAndComment(p, result) + +proc parseBind(p: var TParser, k: TNodeKind): PNode = + #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma + #| mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma + result = newNodeP(k, p) + getTok(p) + optInd(p, result) + while true: + var a = qualifiedIdent(p) + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + +proc parseStmtPragma(p: var TParser): PNode = + result = parsePragma(p) + if p.tok.tokType == tkCurlyLe: + let a = result + result = newNodeI(nkPragmaBlock, a.info) + getTok(p) + skipComment(p, result) + result.add a + result.add parseStmt(p) + eat(p, tkCurlyRi) + +proc simpleStmt(p: var TParser): PNode = + case p.tok.tokType + of tkReturn: result = parseReturnOrRaise(p, nkReturnStmt) + of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt) + of tkYield: result = parseReturnOrRaise(p, nkYieldStmt) + of tkDiscard: result = parseReturnOrRaise(p, nkDiscardStmt) + of tkBreak: result = parseReturnOrRaise(p, nkBreakStmt) + of tkContinue: result = parseReturnOrRaise(p, nkContinueStmt) + of tkCurlyDotLe: result = parseStmtPragma(p) + of tkImport: result = parseImport(p, nkImportStmt) + of tkExport: result = parseImport(p, nkExportStmt) + of tkFrom: result = parseFromStmt(p) + of tkInclude: result = parseIncludeStmt(p) + of tkComment: result = newCommentStmt(p) + else: + if isExprStart(p): result = parseExprStmt(p) + else: result = ast.emptyNode + if result.kind notin {nkEmpty, nkCommentStmt}: skipComment(p, result) + +proc complexOrSimpleStmt(p: var TParser): PNode = + case p.tok.tokType + of tkIf: result = parseIfOrWhen(p, nkIfStmt) + of tkWhile: result = parseWhile(p) + of tkCase: result = parseCase(p) + of tkTry: result = parseTry(p) + of tkFor: result = parseFor(p) + of tkBlock: result = parseBlock(p) + of tkStatic: result = parseStaticOrDefer(p, nkStaticStmt) + of tkDefer: result = parseStaticOrDefer(p, nkDefer) + of tkAsm: result = parseAsm(p) + of tkProc: result = parseRoutine(p, nkProcDef) + of tkMethod: result = parseRoutine(p, nkMethodDef) + of tkIterator: result = parseRoutine(p, nkIteratorDef) + of tkMacro: result = parseRoutine(p, nkMacroDef) + of tkTemplate: result = parseRoutine(p, nkTemplateDef) + of tkConverter: result = parseRoutine(p, nkConverterDef) + of tkType: + getTok(p) + if p.tok.tokType == tkBracketLe: + getTok(p) + result = newNodeP(nkTypeOfExpr, p) + result.addSon(primary(p, pmTypeDesc)) + eat(p, tkBracketRi) + result = parseOperators(p, result, -1, pmNormal) + else: + result = parseSection(p, nkTypeSection, parseTypeDef) + of tkConst: result = parseSection(p, nkConstSection, parseConstant) + of tkLet: result = parseSection(p, nkLetSection, parseVariable) + of tkWhen: result = parseIfOrWhen(p, nkWhenStmt) + of tkVar: result = parseSection(p, nkVarSection, parseVariable) + of tkBind: result = parseBind(p, nkBindStmt) + of tkMixin: result = parseBind(p, nkMixinStmt) + of tkUsing: result = parseSection(p, nkUsingStmt, parseVariable) + else: result = simpleStmt(p) + +proc parseStmt(p: var TParser): PNode = + result = complexOrSimpleStmt(p) proc parseAll*(p: var TParser): PNode = - result = nil + ## Parses the rest of the input stream held by the parser into a PNode. + result = newNodeP(nkStmtList, p) + while p.tok.tokType != tkEof: + var a = complexOrSimpleStmt(p) + if a.kind != nkEmpty: + addSon(result, a) + else: + parMessage(p, errExprExpected, p.tok) + # bugfix: consume a token here to prevent an endless loop: + getTok(p) proc parseTopLevelStmt*(p: var TParser): PNode = - result = nil - + ## Implements an iterator which, when called repeatedly, returns the next + ## top-level statement or emptyNode if end of stream. + result = ast.emptyNode + while true: + case p.tok.tokType + of tkSemiColon: getTok(p) + of tkEof: break + else: + result = complexOrSimpleStmt(p) + if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) + break diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index f4109b26d..e11a8d08b 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -323,7 +323,8 @@ proc processOption(c: PContext, n: PNode): bool = of wStacktrace: onOff(c, n, {optStackTrace}) of wLinetrace: onOff(c, n, {optLineTrace}) of wDebugger: onOff(c, n, {optEndb}) - of wProfiler: onOff(c, n, {optProfiler}) + of wProfiler: onOff(c, n, {optProfiler, optMemTracker}) + of wMemTracker: onOff(c, n, {optMemTracker}) of wByRef: onOff(c, n, {optByRef}) of wDynlib: processDynLib(c, n, nil) of wOptimization: diff --git a/compiler/renderer.nim b/compiler/renderer.nim index a116a8afe..926e67743 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -54,8 +54,6 @@ proc isKeyword*(i: PIdent): bool = (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)): result = true -proc isKeyword*(s: string): bool = isKeyword(getIdent(s)) - proc renderDefinitionName*(s: PSym, noQuotes = false): string = ## Returns the definition name of the symbol. ## @@ -289,8 +287,7 @@ proc lsub(n: PNode): int proc litAux(n: PNode, x: BiggestInt, size: int): string = proc skip(t: PType): PType = result = t - while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, - tyConst, tyMutable}: + while result.kind in {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal}: result = lastSon(result) if n.typ != nil and n.typ.skip.kind in {tyBool, tyEnum}: let enumfields = n.typ.skip.n diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 679e7ba15..f7e5a0f84 100644 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -142,6 +142,7 @@ type methods*: TSymSeq origFile: string inViewMode: bool + cache*: IdentCache PRodReader* = ref TRodReader @@ -219,7 +220,7 @@ proc decodeNodeLazyBody(r: PRodReader, fInfo: TLineInfo, if r.s[r.pos] == '!': inc(r.pos) var fl = decodeStr(r.s, r.pos) - result.ident = getIdent(fl) + result.ident = r.cache.getIdent(fl) else: internalError(result.info, "decodeNode: nkIdent") of nkSym: @@ -401,7 +402,7 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym = internalError(info, "decodeSym: no id") if r.s[r.pos] == '&': inc(r.pos) - ident = getIdent(decodeStr(r.s, r.pos)) + ident = r.cache.getIdent(decodeStr(r.s, r.pos)) else: internalError(info, "decodeSym: no ident") #echo "decoding: {", ident.s @@ -519,7 +520,7 @@ proc newStub(r: PRodReader, name: string, id: int): PSym = new(result) result.kind = skStub result.id = id - result.name = getIdent(name) + result.name = r.cache.getIdent(name) result.position = r.readerIndex setId(id) #MessageOut(result.name.s); if debugIds: registerID(result) @@ -632,7 +633,7 @@ proc processRodFile(r: PRodReader, hash: SecureHash) = while r.s[r.pos] > '\x0A': w = decodeStr(r.s, r.pos) inc(d) - if not condsyms.isDefined(getIdent(w)): + if not condsyms.isDefined(r.cache.getIdent(w)): r.reason = rrDefines #MessageOut('not defined, but should: ' + w); if r.s[r.pos] == ' ': inc(r.pos) if d != countDefinedSymbols(): r.reason = rrDefines @@ -707,8 +708,9 @@ proc startsWith(buf: cstring, token: string, pos = 0): bool = result = s == token.len proc newRodReader(modfilename: string, hash: SecureHash, - readerIndex: int): PRodReader = + readerIndex: int; cache: IdentCache): PRodReader = new(result) + result.cache = cache try: result.memfile = memfiles.open(modfilename) except OSError: @@ -866,7 +868,7 @@ proc getHash*(fileIdx: int32): SecureHash = template growCache*(cache, pos) = if cache.len <= pos: cache.setLen(pos+1) -proc checkDep(fileIdx: int32): TReasonForRecompile = +proc checkDep(fileIdx: int32; cache: IdentCache): TReasonForRecompile = assert fileIdx != InvalidFileIDX growCache gMods, fileIdx if gMods[fileIdx].reason != rrEmpty: @@ -877,7 +879,7 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = gMods[fileIdx].reason = rrNone # we need to set it here to avoid cycles result = rrNone var rodfile = toGeneratedFile(filename.withPackageName, RodExt) - var r = newRodReader(rodfile, hash, fileIdx) + var r = newRodReader(rodfile, hash, fileIdx, cache) if r == nil: result = (if existsFile(rodfile): rrRodInvalid else: rrRodDoesNotExist) else: @@ -888,10 +890,10 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = # NOTE: we need to process the entire module graph so that no ID will # be used twice! However, compilation speed does not suffer much from # this, since results are cached. - var res = checkDep(systemFileIdx) + var res = checkDep(systemFileIdx, cache) if res != rrNone: result = rrModDeps for i in countup(0, high(r.modDeps)): - res = checkDep(r.modDeps[i]) + res = checkDep(r.modDeps[i], cache) if res != rrNone: result = rrModDeps # we cannot break here, because of side-effects of `checkDep` @@ -904,14 +906,14 @@ proc checkDep(fileIdx: int32): TReasonForRecompile = gMods[fileIdx].rd = r gMods[fileIdx].reason = result # now we know better -proc handleSymbolFile*(module: PSym): PRodReader = +proc handleSymbolFile*(module: PSym; cache: IdentCache): PRodReader = let fileIdx = module.fileIdx if optSymbolFiles notin gGlobalOptions: module.id = getID() return nil idgen.loadMaxIds(options.gProjectPath / options.gProjectName) - discard checkDep(fileIdx) + discard checkDep(fileIdx, cache) if gMods[fileIdx].reason == rrEmpty: internalError("handleSymbolFile") result = gMods[fileIdx].rd if result != nil: @@ -1078,7 +1080,7 @@ proc writeType(f: File; t: PType) = f.write("]\n") proc viewFile(rodfile: string) = - var r = newRodReader(rodfile, secureHash(""), 0) + var r = newRodReader(rodfile, secureHash(""), 0, newIdentCache()) if r == nil: rawMessage(errGenerated, "cannot open file (or maybe wrong version):" & rodfile) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index addbdade6..f2422f947 100644 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -16,6 +16,8 @@ import condsyms, ropes, idents, securehash, rodread, passes, importer, idgen, rodutils +from modulegraphs import ModuleGraph + type TRodWriter = object of TPassContext module: PSym @@ -34,6 +36,7 @@ type tstack: TTypeSeq # a stack of types to process files: TStringSeq origFile: string + cache: IdentCache PRodWriter = ref TRodWriter @@ -54,7 +57,7 @@ proc fileIdx(w: PRodWriter, filename: string): int = template filename*(w: PRodWriter): string = w.module.filename -proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = +proc newRodWriter(hash: SecureHash, module: PSym; cache: IdentCache): PRodWriter = new(result) result.sstack = @[] result.tstack = @[] @@ -76,6 +79,7 @@ proc newRodWriter(hash: SecureHash, module: PSym): PRodWriter = result.init = "" result.origFile = module.info.toFullPath result.data = newStringOfCap(12_000) + result.cache = cache proc addModDep(w: PRodWriter, dep: string; info: TLineInfo) = if w.modDeps.len != 0: add(w.modDeps, ' ') @@ -575,15 +579,25 @@ proc process(c: PPassContext, n: PNode): PNode = for i in countup(0, sonsLen(n) - 1): discard process(c, n.sons[i]) #var s = n.sons[namePos].sym #addInterfaceSym(w, s) - of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef, + of nkProcDef, nkIteratorDef, nkConverterDef, nkTemplateDef, nkMacroDef: - var s = n.sons[namePos].sym + let s = n.sons[namePos].sym if s == nil: internalError(n.info, "rodwrite.process") if n.sons[bodyPos] == nil: internalError(n.info, "rodwrite.process: body is nil") if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or sfForward notin s.flags: addInterfaceSym(w, s) + of nkMethodDef: + let s = n.sons[namePos].sym + if s == nil: internalError(n.info, "rodwrite.process") + if n.sons[bodyPos] == nil: + internalError(n.info, "rodwrite.process: body is nil") + if n.sons[bodyPos].kind != nkEmpty or s.magic != mNone or + sfForward notin s.flags: + pushSym(w, s) + processStacks(w, false) + of nkVarSection, nkLetSection, nkConstSection: for i in countup(0, sonsLen(n) - 1): var a = n.sons[i] @@ -621,9 +635,9 @@ proc process(c: PPassContext, n: PNode): PNode = else: discard -proc myOpen(module: PSym): PPassContext = +proc myOpen(g: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = if module.id < 0: internalError("rodwrite: module ID not set") - var w = newRodWriter(module.fileIdx.getHash, module) + var w = newRodWriter(module.fileIdx.getHash, module, cache) rawAddInterfaceSym(w, module) result = w diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index f04cef0ee..1105d3b67 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -11,9 +11,9 @@ ## language. import - ast, modules, passes, passaux, condsyms, + ast, modules, idents, passes, passaux, condsyms, options, nimconf, lists, sem, semdata, llstream, vm, vmdef, commands, msgs, - os, times, osproc, wordrecg, strtabs + os, times, osproc, wordrecg, strtabs, modulegraphs # we support 'cmpIgnoreStyle' natively for efficiency: from strutils import cmpIgnoreStyle, contains @@ -25,9 +25,9 @@ proc listDirs(a: VmArgs, filter: set[PathComponent]) = if kind in filter: result.add path setResult(a, result) -proc setupVM*(module: PSym; scriptName: string): PEvalContext = +proc setupVM*(module: PSym; cache: IdentCache; scriptName: string): PEvalContext = # For Nimble we need to export 'setupVM'. - result = newCtx(module) + result = newCtx(module, cache) result.mode = emRepl registerAdditionalOps(result) @@ -134,9 +134,11 @@ proc setupVM*(module: PSym; scriptName: string): PEvalContext = cbconf selfExe: setResult(a, os.getAppFilename()) -proc runNimScript*(scriptName: string; freshDefines=true) = +proc runNimScript*(cache: IdentCache; scriptName: string; + freshDefines=true) = passes.gIncludeFile = includeModule passes.gImportModule = importModule + let graph = newModuleGraph() if freshDefines: initDefines() defineSymbol("nimscript") @@ -146,15 +148,15 @@ proc runNimScript*(scriptName: string; freshDefines=true) = appendStr(searchPaths, options.libpath) - var m = makeModule(scriptName) + var m = graph.makeModule(scriptName) incl(m.flags, sfMainModule) - vm.globalCtx = setupVM(m, scriptName) + vm.globalCtx = setupVM(m, cache, scriptName) - compileSystemModule() - discard processModule(m, llStreamOpen(scriptName, fmRead), nil) + graph.compileSystemModule(cache) + discard graph.processModule(m, llStreamOpen(scriptName, fmRead), nil, cache) # ensure we load 'system.nim' again for the real non-config stuff! - resetAllModulesHard() + resetSystemArtifacts() vm.globalCtx = nil # do not remove the defined symbols #initDefines() diff --git a/compiler/sem.nim b/compiler/sem.nim index 7768833b3..02c779ef0 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -18,6 +18,8 @@ import evaltempl, patterns, parampatterns, sempass2, nimfix.pretty, semmacrosanity, semparallel, lowerings, pluginsupport, plugins.active +from modulegraphs import ModuleGraph + when defined(nimfix): import nimfix.prettybase @@ -272,7 +274,7 @@ proc tryConstExpr(c: PContext, n: PNode): PNode = msgs.gErrorMax = high(int) try: - result = evalConstExpr(c.module, e) + result = evalConstExpr(c.module, c.cache, e) if result == nil or result.kind == nkEmpty: result = nil else: @@ -293,7 +295,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode = result = getConstExpr(c.module, e) if result == nil: #if e.kind == nkEmpty: globalError(n.info, errConstExprExpected) - result = evalConstExpr(c.module, e) + result = evalConstExpr(c.module, c.cache, e) if result == nil or result.kind == nkEmpty: if e.info != n.info: pushInfoContext(n.info) @@ -364,7 +366,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym, #if c.evalContext == nil: # c.evalContext = c.createEvalContext(emStatic) - result = evalMacroCall(c.module, n, nOrig, sym) + result = evalMacroCall(c.module, c.cache, n, nOrig, sym) if efNoSemCheck notin flags: result = semAfterMacroCall(c, result, sym, flags) popInfoContext() @@ -398,8 +400,8 @@ proc addCodeForGenerics(c: PContext, n: PNode) = addSon(n, prc.ast) c.lastGenericIdx = c.generics.len -proc myOpen(module: PSym): PPassContext = - var c = newContext(module) +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = + var c = newContext(graph, module, cache) if c.p != nil: internalError(module.info, "sem.myOpen") c.semConstExpr = semConstExpr c.semExpr = semExpr @@ -428,8 +430,8 @@ proc myOpen(module: PSym): PPassContext = gNotes = ForeignPackageNotes result = c -proc myOpenCached(module: PSym, rd: PRodReader): PPassContext = - result = myOpen(module) +proc myOpenCached(graph: ModuleGraph; module: PSym; rd: PRodReader): PPassContext = + result = myOpen(graph, module, rd.cache) for m in items(rd.methods): methodDef(m, true) proc isImportSystemStmt(n: PNode): bool = diff --git a/compiler/semasgn.nim b/compiler/semasgn.nim index 2e925e386..c4116a814 100644 --- a/compiler/semasgn.nim +++ b/compiler/semasgn.nim @@ -221,14 +221,15 @@ proc liftBodyAux(c: var TLiftCtx; t: PType; body, x, y: PNode) = body.add newAsgnStmt(x, call) of tyVarargs, tyOpenArray: localError(c.info, errGenerated, "cannot copy openArray") - of tyFromExpr, tyIter, tyProxy, tyBuiltInTypeClass, tyUserTypeClass, + of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass, tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything, - tyMutable, tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt, - tyTypeDesc, tyGenericInvocation, tyBigNum, tyConst, tyForward: + tyGenericParam, tyGenericBody, tyNil, tyExpr, tyStmt, + tyTypeDesc, tyGenericInvocation, tyForward: internalError(c.info, "assignment requested for type: " & typeToString(t)) of tyOrdinal, tyRange, tyGenericInst, tyFieldAccessor, tyStatic, tyVar: liftBodyAux(c, lastSon(t), body, x, y) + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("liftBodyAux") proc newProcType(info: TLineInfo; owner: PSym): PType = result = newType(tyProc, owner) diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 3037a6ecc..ca9b5effb 100644 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -462,7 +462,7 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym = call.add(newIdentNode(fn.name, fn.info)) for i in 1.. <fn.typ.n.len: let param = fn.typ.n.sons[i] - let t = skipTypes(param.typ, abstractVar-{tyTypeDesc}) + let t = skipTypes(param.typ, abstractVar-{tyTypeDesc, tyDistinct}) if t.kind == tyDistinct or param.typ.kind == tyDistinct: hasDistinct = true var x: PType if param.typ.kind == tyVar: diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 30b6e261d..2fec8c757 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -1,7 +1,7 @@ # # # The Nim Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2016 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -13,7 +13,8 @@ import strutils, lists, intsets, options, lexer, ast, astalgo, trees, treetab, wordrecg, ropes, msgs, platform, os, condsyms, idents, renderer, types, extccomp, math, - magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef + magicsys, nversion, nimsets, parser, times, passes, rodread, vmdef, + modulegraphs type TOptionEntry* = object of lists.TListEntry # entries to put on a @@ -106,7 +107,10 @@ type instTypeBoundOp*: proc (c: PContext; dc: PSym; t: PType; info: TLineInfo; op: TTypeAttachedOp; col: int): PSym {.nimcall.} selfName*: PIdent + cache*: IdentCache + graph*: ModuleGraph signatures*: TStrTable + recursiveDep*: string proc makeInstPair*(s: PSym, inst: PInstantiation): TInstantiationPair = result.genericSym = s @@ -116,29 +120,13 @@ proc filename*(c: PContext): string = # the module's filename return c.module.filename -proc newContext*(module: PSym): PContext - -proc lastOptionEntry*(c: PContext): POptionEntry -proc newOptionEntry*(): POptionEntry -proc newLib*(kind: TLibKind): PLib -proc addToLib*(lib: PLib, sym: PSym) -proc makePtrType*(c: PContext, baseType: PType): PType -proc newTypeS*(kind: TTypeKind, c: PContext): PType -proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) - proc scopeDepth*(c: PContext): int {.inline.} = result = if c.currentScope != nil: c.currentScope.depthLevel else: 0 -# owner handling: -proc getCurrOwner*(): PSym -proc pushOwner*(owner: PSym) -proc popOwner*() -# implementation - var gOwners*: seq[PSym] = @[] -proc getCurrOwner(): PSym = +proc getCurrOwner*(): PSym = # owner stack (used for initializing the # owner field of syms) # the documentation comment always gets @@ -146,27 +134,27 @@ proc getCurrOwner(): PSym = # BUGFIX: global array is needed! result = gOwners[high(gOwners)] -proc pushOwner(owner: PSym) = +proc pushOwner*(owner: PSym) = add(gOwners, owner) -proc popOwner() = +proc popOwner*() = var length = len(gOwners) if length > 0: setLen(gOwners, length - 1) else: internalError("popOwner") -proc lastOptionEntry(c: PContext): POptionEntry = +proc lastOptionEntry*(c: PContext): POptionEntry = result = POptionEntry(c.optionStack.tail) proc popProcCon*(c: PContext) {.inline.} = c.p = c.p.next -proc newOptionEntry(): POptionEntry = +proc newOptionEntry*(): POptionEntry = new(result) result.options = gOptions result.defaultCC = ccDefault result.dynlib = nil result.notes = gNotes -proc newContext(module: PSym): PContext = +proc newContext*(graph: ModuleGraph; module: PSym; cache: IdentCache): PContext = new(result) result.ambiguousSymbols = initIntSet() initLinkedList(result.optionStack) @@ -180,6 +168,8 @@ proc newContext(module: PSym): PContext = initStrTable(result.userPragmas) result.generics = @[] result.unknownIdents = initIntSet() + result.cache = cache + result.graph = graph initStrTable(result.signatures) @@ -196,16 +186,19 @@ proc addConverter*(c: PContext, conv: PSym) = proc addPattern*(c: PContext, p: PSym) = inclSym(c.patterns, p) -proc newLib(kind: TLibKind): PLib = +proc newLib*(kind: TLibKind): PLib = new(result) result.kind = kind #initObjectSet(result.syms) -proc addToLib(lib: PLib, sym: PSym) = +proc addToLib*(lib: PLib, sym: PSym) = #if sym.annex != nil and not isGenericRoutine(sym): # LocalError(sym.info, errInvalidPragma) sym.annex = lib -proc makePtrType(c: PContext, baseType: PType): PType = +proc newTypeS*(kind: TTypeKind, c: PContext): PType = + result = newType(kind, getCurrOwner()) + +proc makePtrType*(c: PContext, baseType: PType): PType = result = newTypeS(tyPtr, c) addSonSkipIntLit(result, baseType.assertNotNil) @@ -222,7 +215,7 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType = proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode = let typedesc = makeTypeDesc(c, typ) - let sym = newSym(skType, idAnon, getCurrOwner(), info).linkTo(typedesc) + let sym = newSym(skType, c.cache.idAnon, getCurrOwner(), info).linkTo(typedesc) return newSymNode(sym, info) proc makeTypeFromExpr*(c: PContext, n: PNode): PType = @@ -284,9 +277,6 @@ template rangeHasStaticIf*(t: PType): bool = template getStaticTypeFromRange*(t: PType): PType = t.n[1][0][1].typ -proc newTypeS(kind: TTypeKind, c: PContext): PType = - result = newType(kind, getCurrOwner()) - proc errorType*(c: PContext): PType = ## creates a type representing an error state result = newTypeS(tyError, c) @@ -295,7 +285,7 @@ proc errorNode*(c: PContext, n: PNode): PNode = result = newNodeI(nkEmpty, n.info) result.typ = errorType(c) -proc fillTypeS(dest: PType, kind: TTypeKind, c: PContext) = +proc fillTypeS*(dest: PType, kind: TTypeKind, c: PContext) = dest.kind = kind dest.owner = getCurrOwner() dest.size = - 1 diff --git a/compiler/semdestruct.nim b/compiler/semdestruct.nim index 9ea581f3a..85d106056 100644 --- a/compiler/semdestruct.nim +++ b/compiler/semdestruct.nim @@ -124,7 +124,7 @@ proc instantiateDestructor(c: PContext, typ: PType): PType = # destructor that must be used for the varialbe. # The destructor is either user-defined or automatically # generated by the compiler in a member-wise fashion. - var t = skipTypes(typ, {tyConst, tyMutable}).skipGenericAlias + var t = typ.skipGenericAlias let typeHoldingUserDefinition = if t.kind == tyGenericInst: t.base else: t if typeHoldingUserDefinition.destructor != nil: diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index fbbaaf483..8aaf4f9d8 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -607,12 +607,12 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = call.add(a) #echo "NOW evaluating at compile time: ", call.renderTree if sfCompileTime in callee.flags: - result = evalStaticExpr(c.module, call, c.p.owner) + result = evalStaticExpr(c.module, c.cache, call, c.p.owner) if result.isNil: localError(n.info, errCannotInterpretNodeX, renderTree(call)) else: result = fixupTypeAfterEval(c, result, n) else: - result = evalConstExpr(c.module, call) + result = evalConstExpr(c.module, c.cache, call) if result.isNil: result = n else: result = fixupTypeAfterEval(c, result, n) #if result != n: @@ -620,7 +620,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode = proc semStaticExpr(c: PContext, n: PNode): PNode = let a = semExpr(c, n.sons[0]) - result = evalStaticExpr(c.module, a, c.p.owner) + result = evalStaticExpr(c.module, c.cache, a, c.p.owner) if result.isNil: localError(n.info, errCannotInterpretNodeX, renderTree(n)) result = emptyNode @@ -695,6 +695,22 @@ proc semBracketedMacro(c: PContext; outer, inner: PNode; s: PSym; else: assert(false) return +proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = + result = n + let callee = result.sons[0].sym + case callee.kind + of skMacro: result = semMacroExpr(c, result, orig, callee, flags) + of skTemplate: result = semTemplateExpr(c, result, callee, flags) + else: + semFinishOperands(c, result) + activate(c, result) + fixAbstractType(c, result) + analyseIfAddressTakenInCall(c, result) + if callee.magic != mNone: + result = magicsAfterOverloadResolution(c, result, flags) + if c.inTypeClass == 0: + result = evalAtCompileTime(c, result) + proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = result = nil checkMinSonsLen(n, 1) @@ -773,27 +789,11 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = # See bug #904 of how to trigger it: return result #result = afterCallActions(c, result, nOrig, flags) - fixAbstractType(c, result) - analyseIfAddressTakenInCall(c, result) - if result.sons[0].kind == nkSym and result.sons[0].sym.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) - result = evalAtCompileTime(c, result) - -proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags): PNode = - result = n - let callee = result.sons[0].sym - case callee.kind - of skMacro: result = semMacroExpr(c, result, orig, callee, flags) - of skTemplate: result = semTemplateExpr(c, result, callee, flags) + if result.sons[0].kind == nkSym: + result = afterCallActions(c, result, nOrig, flags) else: - semFinishOperands(c, result) - activate(c, result) fixAbstractType(c, result) analyseIfAddressTakenInCall(c, result) - if callee.magic != mNone: - result = magicsAfterOverloadResolution(c, result, flags) - if c.inTypeClass == 0: - result = evalAtCompileTime(c, result) proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode = # this seems to be a hotspot in the compiler! @@ -861,7 +861,7 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, s = newNodeIT(nkCurly, n.info, setType) for j in countup(0, sonsLen(it) - 2): addSon(s, copyTree(it.sons[j])) var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(inExpr, newSymNode(ast.opContains, n.info)) + addSon(inExpr, newSymNode(opContains, n.info)) addSon(inExpr, s) addSon(inExpr, copyTree(r.sons[0])) addSon(check, inExpr) @@ -874,11 +874,11 @@ proc lookupInRecordAndBuildCheck(c: PContext, n, r: PNode, field: PIdent, check = newNodeI(nkCheckedFieldExpr, n.info) addSon(check, ast.emptyNode) # make space for access node var inExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(inExpr, newSymNode(ast.opContains, n.info)) + addSon(inExpr, newSymNode(opContains, n.info)) addSon(inExpr, s) addSon(inExpr, copyTree(r.sons[0])) var notExpr = newNodeIT(nkCall, n.info, getSysType(tyBool)) - addSon(notExpr, newSymNode(ast.opNot, n.info)) + addSon(notExpr, newSymNode(opNot, n.info)) addSon(notExpr, inExpr) addSon(check, notExpr) return @@ -1551,7 +1551,7 @@ proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym = if isCallExpr(n): var expandedSym = qualifiedLookUp(c, n[0], {checkUndeclared}) if expandedSym == nil: - localError(n.info, errUndeclaredIdentifier, n[0].renderTree) + errorUndeclaredIdentifier(c, n.info, n[0].renderTree) return errorSym(c, n[0]) if expandedSym.kind notin {skMacro, skTemplate}: @@ -1574,9 +1574,9 @@ proc getMagicSym(magic: TMagic): PSym = result = newSym(skProc, getIdent($magic), systemModule, gCodegenLineInfo) result.magic = magic -proc newAnonSym(kind: TSymKind, info: TLineInfo, +proc newAnonSym(c: PContext; kind: TSymKind, info: TLineInfo, owner = getCurrOwner()): PSym = - result = newSym(kind, idAnon, owner, info) + result = newSym(kind, c.cache.idAnon, owner, info) result.flags = {sfGenSym} proc semExpandToAst(c: PContext, n: PNode): PNode = @@ -1648,7 +1648,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode = processQuotations(doBlk.sons[bodyPos], op, quotes, ids) - doBlk.sons[namePos] = newAnonSym(skTemplate, n.info).newSymNode + doBlk.sons[namePos] = newAnonSym(c, skTemplate, n.info).newSymNode if ids.len > 0: doBlk.sons[paramsPos] = newNodeI(nkFormalParams, n.info) doBlk[paramsPos].add getSysSym("stmt").newSymNode # return type diff --git a/compiler/semgnrc.nim b/compiler/semgnrc.nim index b8451865e..ab0ce7c4c 100644 --- a/compiler/semgnrc.nim +++ b/compiler/semgnrc.nim @@ -107,7 +107,7 @@ proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags, var s = searchInScopes(c, ident).skipAlias(n) if s == nil: if ident.id notin ctx.toMixin and withinMixin notin flags: - localError(n.info, errUndeclaredIdentifier, ident.s) + errorUndeclaredIdentifier(c, n.info, ident.s) else: if withinBind in flags: result = symChoice(c, n, s, scClosed) @@ -195,7 +195,7 @@ proc semGenericStmt(c: PContext, n: PNode, if s == nil and withinMixin notin flags and fn.kind in {nkIdent, nkAccQuoted} and considerQuotedIdent(fn).id notin ctx.toMixin: - localError(n.info, errUndeclaredIdentifier, fn.renderTree) + errorUndeclaredIdentifier(c, n.info, fn.renderTree) var first = 0 var mixinContext = false diff --git a/compiler/seminst.nim b/compiler/seminst.nim index d7cad6a2f..e1a65da74 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -58,7 +58,7 @@ iterator instantiateGenericParamList(c: PContext, n: PNode, pt: TIdTable): PSym for i, a in n.pairs: internalAssert a.kind == nkSym var q = a.sym - if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic, tyIter}+tyTypeClasses: + if q.typ.kind notin {tyTypeDesc, tyGenericParam, tyStatic}+tyTypeClasses: continue let symKind = if q.typ.kind == tyStatic: skConst else: skType var s = newSym(symKind, q.name, getCurrOwner(), q.info) @@ -99,7 +99,8 @@ proc genericCacheGet(genericSym: PSym, entry: TInstantiation; proc freshGenSyms(n: PNode, owner, orig: PSym, symMap: var TIdTable) = # we need to create a fresh set of gensym'ed symbols: - if n.kind == nkSym and sfGenSym in n.sym.flags and n.sym.owner == orig: + if n.kind == nkSym and sfGenSym in n.sym.flags and + (n.sym.owner == orig or n.sym.owner.kind == skPackage): let s = n.sym var x = PSym(idTableGet(symMap, s)) if x == nil: diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 806b00db6..e72172c81 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -24,7 +24,7 @@ proc semTypeOf(c: PContext; n: PNode): PNode = result = newNodeI(nkTypeOfExpr, n.info) let typExpr = semExprWithType(c, n, {efInTypeof}) result.add typExpr - result.typ = makeTypeDesc(c, typExpr.typ.skipTypes({tyTypeDesc, tyIter})) + result.typ = makeTypeDesc(c, typExpr.typ.skipTypes({tyTypeDesc})) type SemAsgnMode = enum asgnNormal, noOverloadedSubscript, noOverloadedAsgn @@ -143,7 +143,7 @@ proc semBindSym(c: PContext, n: PNode): PNode = var sc = symChoice(c, id, s, TSymChoiceRule(isMixin.intVal)) result.add(sc) else: - localError(n.sons[1].info, errUndeclaredIdentifier, sl.strVal) + errorUndeclaredIdentifier(c, n.sons[1].info, sl.strVal) proc semShallowCopy(c: PContext, n: PNode, flags: TExprFlags): PNode diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index df9b3f69c..8aa8f15c8 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -204,17 +204,22 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; cycleCheck: var IntSet) = let u = s.gcUnsafetyReason if u != nil and not cycleCheck.containsOrIncl(u.id): let msgKind = if onlyWarning: warnGcUnsafe2 else: errGenerated - if u.kind in {skLet, skVar}: + case u.kind + of skLet, skVar: message(s.info, msgKind, ("'$#' is not GC-safe as it accesses '$#'" & " which is a global using GC'ed memory") % [s.name.s, u.name.s]) - elif u.kind in routineKinds: + of routineKinds: # recursive call *always* produces only a warning so the full error # message is printed: listGcUnsafety(u, true, cycleCheck) message(s.info, msgKind, "'$#' is not GC-safe as it calls '$#'" % [s.name.s, u.name.s]) + of skParam: + message(s.info, msgKind, + "'$#' is not GC-safe as it performs an indirect call via '$#'" % + [s.name.s, u.name.s]) else: internalAssert u.kind == skUnknown message(u.info, msgKind, @@ -721,7 +726,8 @@ proc track(tracked: PEffects, n: PNode) = # and it's not a recursive call: if not (a.kind == nkSym and a.sym == tracked.owner): markSideEffect(tracked, a) - for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) + if a.kind != nkSym or a.sym.magic != mNBindSym: + for i in 1 .. <len(n): trackOperand(tracked, n.sons[i], paramType(op, i)) if a.kind == nkSym and a.sym.magic in {mNew, mNewFinalize, mNewSeq}: # may not look like an assignment, but it is: let arg = n.sons[1] diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index ebcff643f..0c6f6848e 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -524,7 +524,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = addDefer(c, result, v) checkNilable(v) if sfCompileTime in v.flags: hasCompileTime = true - if hasCompileTime: vm.setupCompileTimeVar(c.module, result) + if hasCompileTime: vm.setupCompileTimeVar(c.module, c.cache, result) proc semConst(c: PContext, n: PNode): PNode = result = copyNode(n) @@ -820,7 +820,7 @@ proc semAllTypeSections(c: PContext; n: PNode): PNode = if containsOrIncl(c.includedFiles, f): localError(n.info, errRecursiveDependencyX, f.toFilename) else: - let code = gIncludeFile(c.module, f) + let code = gIncludeFile(c.graph, c.module, f, c.cache) gatherStmts c, code, result excl(c.includedFiles, f) of nkStmtList: @@ -922,7 +922,7 @@ proc semProcAnnotation(c: PContext, prc: PNode; if m == nil: if key.kind == nkIdent and key.ident.id == ord(wDelegator): if considerQuotedIdent(prc.sons[namePos]).s == "()": - prc.sons[namePos] = newIdentNode(idDelegator, prc.info) + prc.sons[namePos] = newIdentNode(c.cache.idDelegator, prc.info) prc.sons[pragmasPos] = copyExcept(n, i) else: localError(prc.info, errOnlyACallOpCanBeDelegator) @@ -965,7 +965,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = checkSonsLen(n, bodyPos + 1) var s: PSym if n[namePos].kind != nkSym: - s = newSym(skProc, idAnon, getCurrOwner(), n.info) + s = newSym(skProc, c.cache.idAnon, getCurrOwner(), n.info) s.ast = n n.sons[namePos] = newSymNode(s) else: @@ -1159,7 +1159,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, assert phase == stepRegisterSymbol if n[namePos].kind == nkEmpty: - s = newSym(kind, idAnon, getCurrOwner(), n.info) + s = newSym(kind, c.cache.idAnon, getCurrOwner(), n.info) incl(s.flags, sfUsed) isAnon = true else: @@ -1418,7 +1418,7 @@ proc evalInclude(c: PContext, n: PNode): PNode = if containsOrIncl(c.includedFiles, f): localError(n.info, errRecursiveDependencyX, f.toFilename) else: - addSon(result, semStmt(c, gIncludeFile(c.module, f))) + addSon(result, semStmt(c, gIncludeFile(c.graph, c.module, f, c.cache))) excl(c.includedFiles, f) proc setLine(n: PNode, info: TLineInfo) = @@ -1445,7 +1445,7 @@ proc semStaticStmt(c: PContext, n: PNode): PNode = #writeStackTrace() let a = semStmt(c, n.sons[0]) n.sons[0] = a - evalStaticStmt(c.module, a, c.p.owner) + evalStaticStmt(c.module, c.cache, a, c.p.owner) result = newNodeI(nkDiscardStmt, n.info, 1) result.sons[0] = emptyNode when false: diff --git a/compiler/service.nim b/compiler/service.nim index 640dd2010..ac04b7860 100644 --- a/compiler/service.nim +++ b/compiler/service.nim @@ -11,7 +11,7 @@ import times, commands, options, msgs, nimconf, - extccomp, strutils, os, platform, parseopt + extccomp, strutils, os, platform, parseopt, idents when useCaas: import net @@ -45,11 +45,11 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = if optRun notin gGlobalOptions and arguments != "" and options.command.normalize != "run": rawMessage(errArgsNeedRunOption, []) -proc serve*(action: proc (){.nimcall.}) = +proc serve*(cache: IdentCache; action: proc (cache: IdentCache){.nimcall.}) = template execute(cmd) = curCaasCmd = cmd processCmdLine(passCmd2, cmd) - action() + action(cache) gErrorCounter = 0 let typ = getConfigVar("server.type") diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index df5a76a57..15171874f 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -549,8 +549,6 @@ proc procTypeRel(c: var TCandidate, f, a: PType): TTypeRelation = of tyNil: result = f.allowsNil - of tyIter: - if tfIterator in f.flags: result = typeRel(c, f.base, a.base) else: discard proc typeRangeRel(f, a: PType): TTypeRelation {.noinline.} = @@ -1021,11 +1019,9 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = var fskip = skippedNone let aobj = x.skipToObject(askip) let fobj = genericBody.lastSon.skipToObject(fskip) + var depth = -1 if fobj != nil and aobj != nil and askip == fskip: - let depth = isObjectSubtype(c, aobj, fobj, f) - if depth >= 0: - c.inheritancePenalty += depth - return if depth == 0: isGeneric else: isSubtype + depth = isObjectSubtype(c, aobj, fobj, f) result = typeRel(c, genericBody, x) if result != isNone: # see tests/generics/tgeneric3.nim for an example that triggers this @@ -1047,6 +1043,11 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: put(c, f.sons[i], x) + if depth >= 0: + c.inheritancePenalty += depth + # bug #4863: We still need to bind generic alias crap, so + # we cannot return immediately: + result = if depth == 0: isGeneric else: isSubtype of tyAnd: considerPreviousT: result = isEqual @@ -1220,13 +1221,6 @@ proc typeRel(c: var TCandidate, f, aOrig: PType, doBind = true): TTypeRelation = else: result = isNone - of tyIter: - if a.kind == tyIter or - (a.kind == tyProc and tfIterator in a.flags): - result = typeRel(c, f.base, a.base) - else: - result = isNone - of tyStmt: if aOrig != nil and tfOldSchoolExprStmt notin f.flags: put(c, f, aOrig) @@ -1584,8 +1578,9 @@ proc prepareOperand(c: PContext; formal: PType; a: PNode): PNode = elif a.typ.isNil: # XXX This is unsound! 'formal' can differ from overloaded routine to # overloaded routine! - let flags = if formal.kind == tyIter: {efDetermineType, efWantIterator} - else: {efDetermineType, efAllowStmt} + let flags = {efDetermineType, efAllowStmt} + #if formal.kind == tyIter: {efDetermineType, efWantIterator} + #else: {efDetermineType, efAllowStmt} #elif formal.kind == tyStmt: {efDetermineType, efWantStmt} #else: {efDetermineType} result = c.semOperand(c, a, flags) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 52f00550b..39689099a 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -233,7 +233,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) = # error: no known module name: typ = nil else: - let m = gImportModule(c.module, fullpath.fileInfoIdx) + let m = gImportModule(c.graph, c.module, fullpath.fileInfoIdx, c.cache) if m == nil: typ = nil else: for it in items(n.sym.tab): diff --git a/compiler/syntaxes.nim b/compiler/syntaxes.nim index 37ea6e2db..4745b1ac7 100644 --- a/compiler/syntaxes.nim +++ b/compiler/syntaxes.nim @@ -26,34 +26,11 @@ const "strip"] type - TParsers*{.final.} = object + TParsers* = object skin*: TParserKind parser*: TParser - -proc parseFile*(fileIdx: int32): PNode{.procvar.} -proc openParsers*(p: var TParsers, fileIdx: int32, inputstream: PLLStream) -proc closeParsers*(p: var TParsers) -proc parseAll*(p: var TParsers): PNode -proc parseTopLevelStmt*(p: var TParsers): PNode - # implements an iterator. Returns the next top-level statement or nil if end - # of stream. - -# implementation - -proc parseFile(fileIdx: int32): PNode = - var - p: TParsers - f: File - let filename = fileIdx.toFullPathConsiderDirty - if not open(f, filename): - rawMessage(errCannotOpenFile, filename) - return - openParsers(p, fileIdx, llStreamOpen(f)) - result = parseAll(p) - closeParsers(p) - -proc parseAll(p: var TParsers): PNode = +proc parseAll*(p: var TParsers): PNode = case p.skin of skinStandard, skinStrongSpaces: result = parser.parseAll(p.parser) @@ -63,7 +40,7 @@ proc parseAll(p: var TParsers): PNode = internalError("parser to implement") result = ast.emptyNode -proc parseTopLevelStmt(p: var TParsers): PNode = +proc parseTopLevelStmt*(p: var TParsers): PNode = case p.skin of skinStandard, skinStrongSpaces: result = parser.parseTopLevelStmt(p.parser) @@ -74,18 +51,18 @@ proc parseTopLevelStmt(p: var TParsers): PNode = result = ast.emptyNode proc utf8Bom(s: string): int = - if (s[0] == '\xEF') and (s[1] == '\xBB') and (s[2] == '\xBF'): + if s[0] == '\xEF' and s[1] == '\xBB' and s[2] == '\xBF': result = 3 else: result = 0 proc containsShebang(s: string, i: int): bool = - if (s[i] == '#') and (s[i + 1] == '!'): + if s[i] == '#' and s[i+1] == '!': var j = i + 2 while s[j] in Whitespace: inc(j) result = s[j] == '/' -proc parsePipe(filename: string, inputStream: PLLStream): PNode = +proc parsePipe(filename: string, inputStream: PLLStream; cache: IdentCache): PNode = result = ast.emptyNode var s = llStreamOpen(filename, fmRead) if s != nil: @@ -101,20 +78,20 @@ proc parsePipe(filename: string, inputStream: PLLStream): PNode = inc(i, 2) while line[i] in Whitespace: inc(i) var q: TParser - openParser(q, filename, llStreamOpen(substr(line, i))) + parser.openParser(q, filename, llStreamOpen(substr(line, i)), cache) result = parser.parseAll(q) - closeParser(q) + parser.closeParser(q) llStreamClose(s) proc getFilter(ident: PIdent): TFilterKind = for i in countup(low(TFilterKind), high(TFilterKind)): - if identEq(ident, filterNames[i]): + if cmpIgnoreStyle(ident.s, filterNames[i]) == 0: return i result = filtNone proc getParser(ident: PIdent): TParserKind = for i in countup(low(TParserKind), high(TParserKind)): - if identEq(ident, parserNames[i]): + if cmpIgnoreStyle(ident.s, parserNames[i]) == 0: return i rawMessage(errInvalidDirectiveX, ident.s) @@ -150,8 +127,7 @@ proc evalPipe(p: var TParsers, n: PNode, filename: string, start: PLLStream): PLLStream = result = start if n.kind == nkEmpty: return - if n.kind == nkInfix and n.sons[0].kind == nkIdent and - identEq(n.sons[0].ident, "|"): + if n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "|": for i in countup(1, 2): if n.sons[i].kind == nkInfix: result = evalPipe(p, n.sons[i], filename, result) @@ -162,18 +138,31 @@ proc evalPipe(p: var TParsers, n: PNode, filename: string, else: result = applyFilter(p, n, filename, result) -proc openParsers(p: var TParsers, fileIdx: int32, inputstream: PLLStream) = +proc openParsers*(p: var TParsers, fileIdx: int32, inputstream: PLLStream; + cache: IdentCache) = var s: PLLStream p.skin = skinStandard let filename = fileIdx.toFullPathConsiderDirty - var pipe = parsePipe(filename, inputstream) + var pipe = parsePipe(filename, inputstream, cache) if pipe != nil: s = evalPipe(p, pipe, filename, inputstream) else: s = inputstream case p.skin of skinStandard, skinBraces, skinEndX: - parser.openParser(p.parser, fileIdx, s, false) + parser.openParser(p.parser, fileIdx, s, cache, false) of skinStrongSpaces: - parser.openParser(p.parser, fileIdx, s, true) + parser.openParser(p.parser, fileIdx, s, cache, true) -proc closeParsers(p: var TParsers) = +proc closeParsers*(p: var TParsers) = parser.closeParser(p.parser) + +proc parseFile*(fileIdx: int32; cache: IdentCache): PNode {.procvar.} = + var + p: TParsers + f: File + let filename = fileIdx.toFullPathConsiderDirty + if not open(f, filename): + rawMessage(errCannotOpenFile, filename) + return + openParsers(p, fileIdx, llStreamOpen(f), cache) + result = parseAll(p) + closeParsers(p) diff --git a/compiler/trees.nim b/compiler/trees.nim index a629b3834..08a1a8c1f 100644 --- a/compiler/trees.nim +++ b/compiler/trees.nim @@ -10,7 +10,7 @@ # tree helper routines import - ast, astalgo, lexer, msgs, strutils, wordrecg + ast, astalgo, lexer, msgs, strutils, wordrecg, idents proc cyclicTreeAux(n: PNode, visited: var seq[PNode]): bool = if n == nil: return diff --git a/compiler/types.nim b/compiler/types.nim index 3db0c4507..fc50449ec 100644 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -51,18 +51,14 @@ const # TODO: Remove tyTypeDesc from each abstractX and (where necessary) # replace with typedescX abstractPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, - tyConst, tyMutable, tyTypeDesc} - abstractInst* = {tyGenericInst, tyDistinct, tyConst, tyMutable, tyOrdinal, tyTypeDesc} + abstractVar* = {tyVar, tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc} + abstractRange* = {tyGenericInst, tyRange, tyDistinct, tyOrdinal, tyTypeDesc} + abstractVarRange* = {tyGenericInst, tyRange, tyVar, tyDistinct, tyOrdinal, + tyTypeDesc} + abstractInst* = {tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc} - skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyConst, tyMutable, - tyTypeDesc} + skipPtrs* = {tyVar, tyPtr, tyRef, tyGenericInst, tyTypeDesc} # typedescX is used if we're sure tyTypeDesc should be included (or skipped) typedescPtrs* = abstractPtrs + {tyTypeDesc} typedescInst* = abstractInst + {tyTypeDesc} @@ -116,8 +112,7 @@ proc isFloatLit*(t: PType): bool {.inline.} = proc isCompatibleToCString(a: PType): bool = if a.kind == tyArray: if (firstOrd(a.sons[0]) == 0) and - (skipTypes(a.sons[0], {tyRange, tyConst, - tyMutable, tyGenericInst}).kind in + (skipTypes(a.sons[0], {tyRange, tyGenericInst}).kind in {tyInt..tyInt64, tyUInt..tyUInt64}) and (a.sons[1].kind == tyChar): result = true @@ -151,13 +146,12 @@ proc isOrdinalType(t: PType): bool = const # caution: uint, uint64 are no ordinal types! baseKinds = {tyChar,tyInt..tyInt64,tyUInt8..tyUInt32,tyBool,tyEnum} - parentKinds = {tyRange, tyOrdinal, tyConst, tyMutable, tyGenericInst, - tyDistinct} + parentKinds = {tyRange, tyOrdinal, tyGenericInst, tyDistinct} t.kind in baseKinds or (t.kind in parentKinds and isOrdinalType(t.sons[0])) proc enumHasHoles(t: PType): bool = var b = t - while b.kind in {tyConst, tyMutable, tyRange, tyGenericInst}: b = b.sons[0] + while b.kind in {tyRange, tyGenericInst}: b = b.sons[0] result = b.kind == tyEnum and tfEnumHasHoles in b.flags proc iterOverTypeAux(marker: var IntSet, t: PType, iter: TTypeIter, @@ -275,7 +269,7 @@ proc analyseObjectWithTypeFieldAux(t: PType, if res == frHeader: result = frHeader if result == frNone: if isObjectWithTypeFieldPredicate(t): result = frHeader - of tyGenericInst, tyDistinct, tyConst, tyMutable: + of tyGenericInst, tyDistinct: result = analyseObjectWithTypeFieldAux(lastSon(t), marker) of tyArray, tyArrayConstr, tyTuple: for i in countup(0, sonsLen(t) - 1): @@ -408,8 +402,8 @@ const "int", "int8", "int16", "int32", "int64", "float", "float32", "float64", "float128", "uint", "uint8", "uint16", "uint32", "uint64", - "bignum", "const ", - "!", "varargs[$1]", "iter[$1]", "Error Type", + "unused0", "unused1", + "unused2", "varargs[$1]", "unused", "Error Type", "BuiltInTypeClass", "UserTypeClass", "UserTypeClassInst", "CompositeTypeClass", "and", "or", "not", "any", "static", "TypeFromExpr", "FieldAccessor", @@ -534,7 +528,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = add(result, typeToString(t.sons[i])) if i < sonsLen(t) - 1: add(result, ", ") add(result, ')') - of tyPtr, tyRef, tyVar, tyMutable, tyConst: + of tyPtr, tyRef, tyVar: result = typeToStr[t.kind] if t.len >= 2: setLen(result, result.len-1) @@ -572,7 +566,7 @@ proc typeToString(typ: PType, prefer: TPreferedDesc = preferName): string = addSep(prag) add(prag, "locks: " & $t.lockLevel) if len(prag) != 0: add(result, "{." & prag & ".}") - of tyVarargs, tyIter: + of tyVarargs: result = typeToStr[t.kind] % typeToString(t.sons[0]) else: result = typeToStr[t.kind] @@ -606,7 +600,7 @@ proc firstOrd(t: PType): BiggestInt = else: assert(t.n.sons[0].kind == nkSym) result = t.n.sons[0].sym.position - of tyGenericInst, tyDistinct, tyConst, tyMutable, tyTypeDesc, tyFieldAccessor: + of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor: result = firstOrd(lastSon(t)) of tyOrdinal: if t.len > 0: result = firstOrd(lastSon(t)) @@ -642,8 +636,7 @@ proc lastOrd(t: PType): BiggestInt = of tyEnum: assert(t.n.sons[sonsLen(t.n) - 1].kind == nkSym) result = t.n.sons[sonsLen(t.n) - 1].sym.position - of tyGenericInst, tyDistinct, tyConst, tyMutable, - tyTypeDesc, tyFieldAccessor: + of tyGenericInst, tyDistinct, tyTypeDesc, tyFieldAccessor: result = lastOrd(lastSon(t)) of tyProxy: result = 0 of tyOrdinal: @@ -656,7 +649,7 @@ proc lastOrd(t: PType): BiggestInt = proc lengthOrd(t: PType): BiggestInt = case t.kind of tyInt64, tyInt32, tyInt: result = lastOrd(t) - of tyDistinct, tyConst, tyMutable: result = lengthOrd(t.sons[0]) + of tyDistinct: result = lengthOrd(t.sons[0]) else: let last = lastOrd t let first = firstOrd t @@ -925,7 +918,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = case a.kind of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCString, - tyInt..tyBigNum, tyStmt, tyExpr, tyVoid: + tyInt..tyUInt64, tyStmt, tyExpr, tyVoid: result = sameFlags(a, b) of tyStatic, tyFromExpr: result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b) @@ -965,8 +958,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = result = a.sym.position == b.sym.position of tyGenericInvocation, tyGenericBody, tySequence, tyOpenArray, tySet, tyRef, tyPtr, tyVar, tyArrayConstr, - tyArray, tyProc, tyConst, tyMutable, tyVarargs, tyIter, - tyOrdinal, tyTypeClasses, tyFieldAccessor: + tyArray, tyProc, tyVarargs, tyOrdinal, tyTypeClasses, tyFieldAccessor: cycleCheck() if a.kind == tyUserTypeClass and a.n != nil: return a.n == b.n result = sameChildrenAux(a, b, c) and sameFlags(a, b) @@ -980,6 +972,7 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool = sameValue(a.n.sons[1], b.n.sons[1]) of tyGenericInst: discard of tyNone: result = false + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("sameFlags") proc sameBackendType*(x, y: PType): bool = var c = initSameTypeClosure() @@ -1120,7 +1113,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, result = t of tyNil: if kind != skConst: result = t - of tyString, tyBool, tyChar, tyEnum, tyInt..tyBigNum, tyCString, tyPointer: + of tyString, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString, tyPointer: result = nil of tyOrdinal: if kind != skParam: result = t @@ -1143,7 +1136,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, else: result = typeAllowedAux(marker, t.lastSon, skVar, flags+{taHeap}) of tyPtr: result = typeAllowedAux(marker, t.lastSon, skVar, flags+{taHeap}) - of tyArrayConstr, tySet, tyConst, tyMutable, tyIter: + of tyArrayConstr, tySet: for i in countup(0, sonsLen(t) - 1): result = typeAllowedAux(marker, t.sons[i], kind, flags) if result != nil: break @@ -1160,6 +1153,7 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind, # for now same as error node; we say it's a valid type as it should # prevent cascading errors: result = nil + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("typeAllowedAux") proc typeAllowed*(t: PType, kind: TSymKind): PType = # returns 'nil' on success and otherwise the part of the type that is @@ -1250,8 +1244,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = if typ.callConv == ccClosure: result = 2 * ptrSize else: result = ptrSize a = ptrSize - of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray, - tyBigNum: + of tyNil, tyCString, tyString, tySequence, tyPtr, tyRef, tyVar, tyOpenArray: let base = typ.lastSon if base == typ or (base.kind == tyTuple and base.size==szIllegalRecursion): result = szIllegalRecursion @@ -1311,7 +1304,7 @@ proc computeSizeAux(typ: PType, a: var BiggestInt): BiggestInt = if result < 0: return if a < maxAlign: a = maxAlign result = align(result, a) - of tyGenericInst, tyDistinct, tyGenericBody, tyMutable, tyConst, tyIter: + of tyGenericInst, tyDistinct, tyGenericBody: result = computeSizeAux(lastSon(typ), a) of tyTypeDesc: result = computeSizeAux(typ.base, a) @@ -1374,7 +1367,7 @@ proc safeInheritanceDiff*(a, b: PType): int = if a.kind == tyError or b.kind == tyError: result = -1 else: - result = inheritanceDiff(a, b) + result = inheritanceDiff(a.skipTypes(skipPtrs), b.skipTypes(skipPtrs)) proc compatibleEffectsAux(se, re: PNode): bool = if re.isNil: return false diff --git a/compiler/typesrenderer.nim b/compiler/typesrenderer.nim index 438744b1c..e9c27ac9d 100644 --- a/compiler/typesrenderer.nim +++ b/compiler/typesrenderer.nim @@ -37,14 +37,20 @@ proc renderType(n: PNode): string = of nkIdent: result = n.ident.s of nkSym: result = typeToString(n.sym.typ) of nkVarTy: - assert len(n) == 1 - result = renderType(n[0]) + if n.len == 1: + result = renderType(n[0]) + else: + result = "var" of nkRefTy: - assert len(n) == 1 - result = "ref." & renderType(n[0]) + if n.len == 1: + result = "ref." & renderType(n[0]) + else: + result = "ref" of nkPtrTy: - assert len(n) == 1 - result = "ptr." & renderType(n[0]) + if n.len == 1: + result = "ptr." & renderType(n[0]) + else: + result = "ptr" of nkProcTy: assert len(n) > 1 let params = n[0] diff --git a/compiler/vm.nim b/compiler/vm.nim index efcc55c59..1bb440c6c 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -24,6 +24,8 @@ import from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate +from modulegraphs import ModuleGraph + when hasFFI: import evalffi @@ -76,12 +78,13 @@ proc stackTraceAux(c: PCtx; x: PStackFrame; pc: int; recursionLimit=100) = msgWriteln(s) proc stackTrace(c: PCtx, tos: PStackFrame, pc: int, - msg: TMsgKind, arg = "") = + msg: TMsgKind, arg = "", n: PNode = nil) = msgWriteln("stack trace: (most recent call last)") stackTraceAux(c, tos, pc) # XXX test if we want 'globalError' for every mode - if c.mode == emRepl: globalError(c.debug[pc], msg, arg) - else: localError(c.debug[pc], msg, arg) + let lineInfo = if n == nil: c.debug[pc] else: n.info + if c.mode == emRepl: globalError(lineInfo, msg, arg) + else: localError(lineInfo, msg, arg) proc bailOut(c: PCtx; tos: PStackFrame) = stackTrace(c, tos, c.exceptionInstr, errUnhandledExceptionX, @@ -900,7 +903,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = if newPc < pc: handleJmpBack() #echo "new pc ", newPc, " calling: ", prc.name.s var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos) - newSeq(newFrame.slots, prc.offset) + newSeq(newFrame.slots, prc.offset+ord(isClosure)) if not isEmptyType(prc.typ.sons[0]) or prc.kind == skMacro: putIntoReg(newFrame.slots[0], getNullValue(prc.typ.sons[0], prc.info)) for i in 1 .. rc-1: @@ -1242,9 +1245,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = createStr regs[ra] regs[ra].node.strVal = opGorge(regs[rb].node.strVal, - regs[rc].node.strVal, regs[rd].node.strVal) + regs[rc].node.strVal, regs[rd].node.strVal, + c.debug[pc]) of opcNError: - stackTrace(c, tos, pc, errUser, regs[ra].node.strVal) + decodeB(rkNode) + let a = regs[ra].node + let b = regs[rb].node + stackTrace(c, tos, pc, errUser, a.strVal, if b.kind == nkNilLit: nil else: b) of opcNWarning: message(c.debug[pc], warnUser, regs[ra].node.strVal) of opcNHint: @@ -1253,7 +1260,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = decodeB(rkNode) # c.debug[pc].line.int - countLines(regs[rb].strVal) ? var error: string - let ast = parseString(regs[rb].node.strVal, c.debug[pc].toFullPath, + let ast = parseString(regs[rb].node.strVal, c.cache, c.debug[pc].toFullPath, c.debug[pc].line.int, proc (info: TLineInfo; msg: TMsgKind; arg: string) = if error.isNil and msg <= msgs.errMax: @@ -1267,7 +1274,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = of opcParseStmtToAst: decodeB(rkNode) var error: string - let ast = parseString(regs[rb].node.strVal, c.debug[pc].toFullPath, + let ast = parseString(regs[rb].node.strVal, c.cache, c.debug[pc].toFullPath, c.debug[pc].line.int, proc (info: TLineInfo; msg: TMsgKind; arg: string) = if error.isNil and msg <= msgs.errMax: @@ -1421,7 +1428,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = else: regs[rc].node.strVal if k < 0 or k > ord(high(TSymKind)): internalError(c.debug[pc], "request to create symbol of invalid kind") - var sym = newSym(k.TSymKind, name.getIdent, c.module, c.debug[pc]) + var sym = newSym(k.TSymKind, name.getIdent, c.module.owner, c.debug[pc]) incl(sym.flags, sfGenSym) regs[ra].node = newSymNode(sym) of opcTypeTrait: @@ -1504,20 +1511,20 @@ include vmops var globalCtx*: PCtx -proc setupGlobalCtx(module: PSym) = +proc setupGlobalCtx(module: PSym; cache: IdentCache) = if globalCtx.isNil: - globalCtx = newCtx(module) + globalCtx = newCtx(module, cache) registerAdditionalOps(globalCtx) else: refresh(globalCtx, module) -proc myOpen(module: PSym): PPassContext = +proc myOpen(graph: ModuleGraph; module: PSym; cache: IdentCache): PPassContext = #var c = newEvalContext(module, emRepl) #c.features = {allowCast, allowFFI, allowInfiniteLoops} #pushStackFrame(c, newStackFrame()) # XXX produce a new 'globals' environment here: - setupGlobalCtx(module) + setupGlobalCtx(module, cache) result = globalCtx when hasFFI: globalCtx.features = {allowFFI, allowCast} @@ -1535,9 +1542,10 @@ proc myProcess(c: PPassContext, n: PNode): PNode = const evalPass* = makePass(myOpen, nil, myProcess, myProcess) -proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode = +proc evalConstExprAux(module: PSym; cache: IdentCache; prc: PSym, n: PNode, + mode: TEvalMode): PNode = let n = transformExpr(module, n) - setupGlobalCtx(module) + setupGlobalCtx(module, cache) var c = globalCtx let oldMode = c.mode defer: c.mode = oldMode @@ -1552,17 +1560,17 @@ proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode = result = rawExecute(c, start, tos).regToNode if result.info.line < 0: result.info = n.info -proc evalConstExpr*(module: PSym, e: PNode): PNode = - result = evalConstExprAux(module, nil, e, emConst) +proc evalConstExpr*(module: PSym; cache: IdentCache, e: PNode): PNode = + result = evalConstExprAux(module, cache, nil, e, emConst) -proc evalStaticExpr*(module: PSym, e: PNode, prc: PSym): PNode = - result = evalConstExprAux(module, prc, e, emStaticExpr) +proc evalStaticExpr*(module: PSym; cache: IdentCache, e: PNode, prc: PSym): PNode = + result = evalConstExprAux(module, cache, prc, e, emStaticExpr) -proc evalStaticStmt*(module: PSym, e: PNode, prc: PSym) = - discard evalConstExprAux(module, prc, e, emStaticStmt) +proc evalStaticStmt*(module: PSym; cache: IdentCache, e: PNode, prc: PSym) = + discard evalConstExprAux(module, cache, prc, e, emStaticStmt) -proc setupCompileTimeVar*(module: PSym, n: PNode) = - discard evalConstExprAux(module, nil, n, emStaticStmt) +proc setupCompileTimeVar*(module: PSym; cache: IdentCache, n: PNode) = + discard evalConstExprAux(module, cache, nil, n, emStaticStmt) proc setupMacroParam(x: PNode, typ: PType): TFullReg = case typ.kind @@ -1581,7 +1589,8 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg = var evalMacroCounter: int -proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = +proc evalMacroCall*(module: PSym; cache: IdentCache, n, nOrig: PNode, + sym: PSym): PNode = # XXX globalError() is ugly here, but I don't know a better solution for now inc(evalMacroCounter) if evalMacroCounter > 100: @@ -1594,7 +1603,7 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode = n.renderTree, $ <n.safeLen, $ <sym.typ.len]) - setupGlobalCtx(module) + setupGlobalCtx(module, cache) var c = globalCtx c.callsite = nOrig diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index 83c1dbf43..7fb35e890 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -10,7 +10,7 @@ ## This module contains the type definitions for the new evaluation engine. ## An instruction is 1-3 int32s in memory, it is a register based VM. -import ast, passes, msgs, intsets +import ast, passes, msgs, idents, intsets const byteExcess* = 128 # we use excess-K for immediates @@ -203,16 +203,18 @@ type comesFromHeuristic*: TLineInfo # Heuristic for better macro stack traces callbacks*: seq[tuple[key: string, value: VmCallback]] errorFlag*: string + cache*: IdentCache TPosition* = distinct int PEvalContext* = PCtx -proc newCtx*(module: PSym): PCtx = +proc newCtx*(module: PSym; cache: IdentCache): PCtx = PCtx(code: @[], debug: @[], globals: newNode(nkStmtListExpr), constants: newNode(nkStmtList), types: @[], prc: PProc(blocks: @[]), module: module, loopIterations: MaxLoopIterations, - comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "") + comesFromHeuristic: unknownLineInfo(), callbacks: @[], errorFlag: "", + cache: cache) proc refresh*(c: PCtx, module: PSym) = c.module = module diff --git a/compiler/vmdeps.nim b/compiler/vmdeps.nim index b40ca1058..bd6908722 100644 --- a/compiler/vmdeps.nim +++ b/compiler/vmdeps.nim @@ -19,7 +19,8 @@ proc readOutput(p: Process): string = result.setLen(result.len - "\n".len) discard p.waitForExit -proc opGorge*(cmd, input, cache: string): string = +proc opGorge*(cmd, input, cache: string, info: TLineInfo): string = + let workingDir = parentDir(info.toFullPath) if cache.len > 0:# and optForceFullMake notin gGlobalOptions: let h = secureHash(cmd & "\t" & input & "\t" & cache) let filename = options.toGeneratedFile("gorge_" & $h, "txt") @@ -30,7 +31,8 @@ proc opGorge*(cmd, input, cache: string): string = return var readSuccessful = false try: - var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout}) + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) if input.len != 0: p.inputStream.write(input) p.inputStream.close() @@ -41,7 +43,8 @@ proc opGorge*(cmd, input, cache: string): string = if not readSuccessful: result = "" else: try: - var p = startProcess(cmd, options={poEvalCommand, poStderrToStdout}) + var p = startProcess(cmd, workingDir, + options={poEvalCommand, poStderrToStdout}) if input.len != 0: p.inputStream.write(input) p.inputStream.close() @@ -266,11 +269,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; of tyUInt16: result = atomicType("uint16", mUint16) of tyUInt32: result = atomicType("uint32", mUint32) of tyUInt64: result = atomicType("uint64", mUint64) - of tyBigNum: result = atomicType("bignum", mNone) - of tyConst: result = mapTypeToBracket("const", mNone, t, info) - of tyMutable: result = mapTypeToBracket("mutable", mNone, t, info) of tyVarargs: result = mapTypeToBracket("varargs", mVarargs, t, info) - of tyIter: result = mapTypeToBracket("iter", mNone, t, info) of tyProxy: result = atomicType("error", mNone) of tyBuiltInTypeClass: result = mapTypeToBracket("builtinTypeClass", mNone, t, info) @@ -292,6 +291,7 @@ proc mapTypeToAstX(t: PType; info: TLineInfo; result.add atomicType("static", mNone) if t.n != nil: result.add t.n.copyTree + of tyUnused, tyUnused0, tyUnused1, tyUnused2: internalError("mapTypeToAstX") proc opMapTypeToAst*(t: PType; info: TLineInfo): PNode = result = mapTypeToAstX(t, info, false, true) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 6bfc33f00..ed8f3f338 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1068,7 +1068,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = else: # setter unused(n, dest) - genUnaryStmt(c, n, opcNError) + genBinaryStmt(c, n, opcNError) of mNCallSite: if dest < 0: dest = c.getTemp(n.typ) c.gABC(n, opcCallSite, dest) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index b5ffd51c2..cf66b6358 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -13,8 +13,7 @@ # does not support strings. Without this the code would # be slow and unreadable. -import - hashes, strutils, idents +from strutils import cmpIgnoreStyle # Keywords must be kept sorted and within a range @@ -35,7 +34,7 @@ type wColon, wColonColon, wEquals, wDot, wDotDot, wStar, wMinus, - wMagic, wThread, wFinal, wProfiler, wObjChecks, + wMagic, wThread, wFinal, wProfiler, wMemTracker, wObjChecks, wIntDefine, wStrDefine, wDestroy, @@ -122,7 +121,7 @@ const ":", "::", "=", ".", "..", "*", "-", - "magic", "thread", "final", "profiler", "objchecks", "intdefine", "strdefine", + "magic", "thread", "final", "profiler", "memtracker", "objchecks", "intdefine", "strdefine", "destroy", @@ -180,17 +179,3 @@ proc findStr*(a: openArray[string], s: string): int = if cmpIgnoreStyle(a[i], s) == 0: return i result = - 1 - -proc whichKeyword*(id: PIdent): TSpecialWord = - if id.id < 0: result = wInvalid - else: result = TSpecialWord(id.id) - -proc whichKeyword*(id: string): TSpecialWord = - result = whichKeyword(getIdent(id)) - -proc initSpecials() = - # initialize the keywords: - for s in countup(succ(low(specialWords)), high(specialWords)): - getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s) - -initSpecials() diff --git a/config/nim.cfg b/config/nim.cfg index 8c8270f3e..b5bcc4053 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -177,10 +177,26 @@ clang.options.speed = "-O3" clang.options.size = "-Os" # Configuration for the Visual C/C++ compiler: -vcc.options.linker = "/DEBUG /Zi /Fd\"$projectName.pdb\" /F33554432" # set the stack size to 8 MB -vcc.options.debug = "/Zi /Fd\"$projectName.pdb\"" +vcc.exe = "vccexe.exe" +vcc.linkerexe = "vccexe.exe" + +# set the options for specific platforms: +@if i386: +vcc.options.always = "--platform:x86 /nologo" +vcc.options.linker = "--platform:x86 /nologo /DEBUG /Zi /F33554432" # set the stack size to 8 MB +@elif amd64: +vcc.options.always = "--platform:amd64 /nologo" +vcc.options.linker = "--platform:amd64 /nologo /DEBUG /Zi /F33554432" # set the stack size to 8 MB +@elif arm: +vcc.options.always = "--platform:arm /nologo" +vcc.options.linker = "--platform:arm /nologo /DEBUG /Zi /F33554432" # set the stack size to 8 MB +@else: vcc.options.always = "/nologo" -vcc.options.speed = "/O2 /arch:SSE2" +vcc.options.linker = "/nologo /DEBUG /Zi /F33554432" # set the stack size to 8 MB +@end + +vcc.options.debug = "/Zi /FS /Od" +vcc.options.speed = "/O2" vcc.options.size = "/O1" # Configuration for the Tiny C Compiler: diff --git a/contributors.txt b/contributors.txt deleted file mode 100644 index d54ec844a..000000000 --- a/contributors.txt +++ /dev/null @@ -1,12 +0,0 @@ -Comex -Eric Doughty-Papassideris -Simon Hafner -Keita Haga -Grzegorz Adam Hankiewicz -Philippe Lhoste -Zahary Karadjov -Mario Ray Mahardhika -Alexander Mitchell-Robinson (Amrykid) -Dominik Picheta -Jonathan Plona -Alexander Rødseth diff --git a/doc/advopt.txt b/doc/advopt.txt index b8980fa9c..991f06397 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -61,6 +61,9 @@ Advanced options: --taintMode:on|off turn taint mode on|off --implicitStatic:on|off turn implicit compile time evaluation on|off --patterns:on|off turn pattern matching on|off + --memTracker:on|off turn memory tracker on|off + --excessiveStackTrace:on|off + stack traces use full file paths --skipCfg do not read the general configuration file --skipUserCfg do not read the user's configuration file --skipParentCfg do not read the parent dirs' configuration files diff --git a/doc/astspec.txt b/doc/astspec.txt index 35eb1727b..f430677af 100644 --- a/doc/astspec.txt +++ b/doc/astspec.txt @@ -1184,6 +1184,7 @@ Nim type Corresponding AST ``proc`` ``nnkProcTy`` ``iterator`` ``nnkIteratorTy`` ``object`` ``nnkObjectTy`` +------------- --------------------------------------------- Take special care when declaring types as ``proc``. The behavior is similar to ``Procedure declaration``, below, but does not treat ``nnkGenericParams``. diff --git a/doc/basicopt.txt b/doc/basicopt.txt index 9a1cfd956..eabd531a1 100644 --- a/doc/basicopt.txt +++ b/doc/basicopt.txt @@ -5,7 +5,7 @@ Command: //compile, c compile project with default code generator (C) //doc generate the documentation for inputfile - //doc2 generate the documentation for the whole project + //doc2 generate the documentation for inputfile Arguments: arguments are passed to the program being run (if --run option is selected) diff --git a/contributing.rst b/doc/contributing.rst index 31f04a5e0..31f04a5e0 100644 --- a/contributing.rst +++ b/doc/contributing.rst diff --git a/doc/docs.txt b/doc/docs.rst index 4484784ae..4484784ae 100644 --- a/doc/docs.txt +++ b/doc/docs.rst diff --git a/docstyle.rst b/doc/docstyle.rst index d789b1df9..d789b1df9 100644 --- a/docstyle.rst +++ b/doc/docstyle.rst diff --git a/doc/gc.rst b/doc/gc.rst index 1c8cb9122..bb0088617 100644 --- a/doc/gc.rst +++ b/doc/gc.rst @@ -19,8 +19,10 @@ This document describes how the GC works and how to tune it for The basic algorithm is *Deferred Reference Counting* with cycle detection. References on the stack are not counted for better performance (and easier C -code generation). The GC **never** scans the whole heap but it may scan the -delta-subgraph of the heap that changed since its last run. +code generation). Cycle detection is currently done by a simple mark&sweep +GC that has to scan the full (thread local heap). ``--gc:v2`` replaces this +with an incremental mark and sweep. That it is not production ready yet, +however. The GC is only triggered in a memory allocation operation. It is not triggered @@ -34,17 +36,7 @@ Cycle collector =============== The cycle collector can be en-/disabled independently from the other parts of -the GC with ``GC_enableMarkAndSweep`` and ``GC_disableMarkAndSweep``. The -compiler analyses the types for their possibility to build cycles, but often -it is necessary to help this analysis with the ``acyclic`` pragma (see -`acyclic <manual.html#acyclic-pragma>`_ for further information). - -You can also use the ``acyclic`` pragma for data that is cyclic in reality and -then break up the cycles explicitly with ``GC_addCycleRoot``. This can be a -very valuable optimization; the Nim compiler itself relies on this -optimization trick to improve performance. Note that ``GC_addCycleRoot`` is -a quick operation; the root is only registered for the next run of the -cycle collector. +the GC with ``GC_enableMarkAndSweep`` and ``GC_disableMarkAndSweep``. Realtime support @@ -55,19 +47,19 @@ defined via ``--define:useRealtimeGC`` (you can put this into your config file as well). With this switch the GC supports the following operations: .. code-block:: nim - proc GC_setMaxPause*(MaxPauseInUs: int) + proc GC_setMaxPause*(maxPauseInUs: int) proc GC_step*(us: int, strongAdvice = false, stackSize = -1) -The unit of the parameters ``MaxPauseInUs`` and ``us`` is microseconds. +The unit of the parameters ``maxPauseInUs`` and ``us`` is microseconds. These two procs are the two modus operandi of the realtime GC: (1) GC_SetMaxPause Mode You can call ``GC_SetMaxPause`` at program startup and then each triggered - GC run tries to not take longer than ``MaxPause`` time. However, it is + GC run tries to not take longer than ``maxPause`` time. However, it is possible (and common) that the work is nevertheless not evenly distributed - as each call to ``new`` can trigger the GC and thus take ``MaxPause`` + as each call to ``new`` can trigger the GC and thus take ``maxPause`` time. (2) GC_step Mode @@ -86,8 +78,8 @@ These two procs are the two modus operandi of the realtime GC: These procs provide a "best effort" realtime guarantee; in particular the cycle collector is not aware of deadlines yet. Deactivate it to get more predictable realtime behaviour. Tests show that a 2ms max pause -time will be met in almost all cases on modern CPUs unless the cycle collector -is triggered. +time will be met in almost all cases on modern CPUs (with the cycle collector +disabled). Time measurement diff --git a/doc/intern.txt b/doc/intern.txt index 05847169f..d0aaa283a 100644 --- a/doc/intern.txt +++ b/doc/intern.txt @@ -459,7 +459,7 @@ This should produce roughly this code: PEnv = ref object x: int # data - proc anon(y: int, c: PClosure): int = + proc anon(y: int, c: PEnv): int = return y + c.x proc add(x: int): tuple[prc, data] = diff --git a/doc/lib.rst b/doc/lib.rst index 828889968..8bb602b78 100644 --- a/doc/lib.rst +++ b/doc/lib.rst @@ -77,8 +77,9 @@ Collections and algorithms * `lists <lists.html>`_ Nim linked list support. Contains singly and doubly linked lists and circular lists ("rings"). -* `queues <queues.html>`_ - Implementation of a queue. The underlying implementation uses a ``seq``. +* `deques <deques.html>`_ + Implementation of a double-ended queue. + The underlying implementation uses a ``seq``. * `intsets <intsets.html>`_ Efficient implementation of a set of ints as a sparse bit set. * `critbits <critbits.html>`_ diff --git a/doc/manual/exceptions.txt b/doc/manual/exceptions.txt index d06c13df4..0f1240a4a 100644 --- a/doc/manual/exceptions.txt +++ b/doc/manual/exceptions.txt @@ -147,7 +147,7 @@ the ``raise`` statement is the only way to raise an exception. If no exception name is given, the current exception is `re-raised`:idx:. The `ReraiseError`:idx: exception is raised if there is no exception to re-raise. It follows that the ``raise`` statement *always* raises an -exception (unless a raise hook has been provided). +exception. Exception hierarchy diff --git a/doc/manual/types.txt b/doc/manual/types.txt index 02426e0d9..c81bc042b 100644 --- a/doc/manual/types.txt +++ b/doc/manual/types.txt @@ -716,7 +716,8 @@ untraced references are *unsafe*. However for certain low-level operations (accessing the hardware) untraced references are unavoidable. Traced references are declared with the **ref** keyword, untraced references -are declared with the **ptr** keyword. +are declared with the **ptr** keyword. In general, a `ptr T` is implicitly +convertible to the `pointer` type. An empty subscript ``[]`` notation can be used to derefer a reference, the ``addr`` procedure returns the address of an item. An address is always diff --git a/doc/nimc.rst b/doc/nimc.rst index eb1beb549..5d9ed03ab 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -258,6 +258,10 @@ Define Effect ``ssl`` Enables OpenSSL support for the sockets module. ``memProfiler`` Enables memory profiling for the native GC. ``uClibc`` Use uClibc instead of libc. (Relevant for Unix-like OSes) +``checkAbi`` When using types from C headers, add checks that compare + what's in the Nim file with what's in the C header + (requires a C compiler with _Static_assert support, like + any C11 compiler) ================== ========================================================= diff --git a/doc/nims.rst b/doc/nims.rst index 7c76efe42..12d86a905 100644 --- a/doc/nims.rst +++ b/doc/nims.rst @@ -75,22 +75,8 @@ done: Nimble integration ================== -A ``project.nims`` file can also be used as an alternative to -a ``project.nimble`` file to specify the meta information (for example, author, -description) and dependencies of a Nimble package. This means you can easily -have platform specific dependencies: - -.. code-block:: nim - - version = "1.0" - author = "The green goo." - description = "Lexer generation and regex implementation for Nim." - license = "MIT" - - when defined(windows): - requires "oldwinapi >= 1.0" - else: - requires "gtk2 >= 1.0" +See the `Nimble readme <https://github.com/nim-lang/nimble#readme>`_ +for more information. diff --git a/doc/overview.txt b/doc/overview.rst index 5b41752ae..e01520d7c 100644 --- a/doc/overview.txt +++ b/doc/overview.rst @@ -5,5 +5,5 @@ Nim Documentation Overview :Author: Andreas Rumpf :Version: |nimversion| -.. include:: ../doc/docs.txt +.. include:: docs.rst diff --git a/icons/koch-amd64-windows-vcc.res b/icons/koch-amd64-windows-vcc.res new file mode 100644 index 000000000..90d7d1f7a --- /dev/null +++ b/icons/koch-amd64-windows-vcc.res Binary files differdiff --git a/icons/koch-i386-windows-vcc.res b/icons/koch-i386-windows-vcc.res new file mode 100644 index 000000000..90d7d1f7a --- /dev/null +++ b/icons/koch-i386-windows-vcc.res Binary files differdiff --git a/icons/nim-amd64-windows-vcc.res b/icons/nim-amd64-windows-vcc.res new file mode 100644 index 000000000..b2d8fc9eb --- /dev/null +++ b/icons/nim-amd64-windows-vcc.res Binary files differdiff --git a/icons/nim-i386-windows-vcc.res b/icons/nim-i386-windows-vcc.res new file mode 100644 index 000000000..b2d8fc9eb --- /dev/null +++ b/icons/nim-i386-windows-vcc.res Binary files differdiff --git a/install.txt b/install.txt index 833fbf0fb..6ab562b07 100644 --- a/install.txt +++ b/install.txt @@ -33,7 +33,8 @@ is more cumbersome then. To complete the installation you should also build Nim's tools like ``nimsuggest``, ``nimble`` or ``nimgrep`` via:: - nim e install_tools.nims + nim c koch + koch tools Note that these tools should also end up in your ``PATH`` so adding ``$your_install_dir/bin/nim`` to your ``PATH`` is preferred over the symlink @@ -53,10 +54,8 @@ or clang. Installation on Windows ----------------------- -Install Nim by downloading and running the ``nim_$version.exe`` file. -As default, the ``GCC`` compiler is used that is bundled with this installer. -You can change the configuration file ``config/nim.cfg`` to use -another C compiler or change the path to GCC. +Install Nim by downloading and unzipping the ``nim_$version.zip`` file. +Run ``finish.exe`` to detect and setup your MingW environment. Currently, the following C compilers are supported under Windows: @@ -85,4 +84,5 @@ Nimble is Nim's package manager. For the source based installations where you added Nim's ``bin`` directory to your ``$PATH`` the easiest way of installing Nimble is via:: - nim e install_nimble.nims + nim c koch + koch nimble diff --git a/install_nimble.nims b/install_nimble.nims index 05024c046..6e929f499 100644 --- a/install_nimble.nims +++ b/install_nimble.nims @@ -3,6 +3,8 @@ import ospaths mode = ScriptMode.Verbose +echo "This script is deprecated. Use 'koch nimble' instead." + var id = 0 while dirExists("nimble" & $id): inc id @@ -20,4 +22,4 @@ try: mvFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe except OSError: cpFile "nimble" & $id & "/src/nimble".toExe, "bin/nimble".toExe - + diff --git a/install_tools.nims b/install_tools.nims index e211ff491..f5f320f78 100644 --- a/install_tools.nims +++ b/install_tools.nims @@ -3,6 +3,8 @@ import ospaths mode = ScriptMode.Verbose +echo "This script is deprecated. Use 'koch tools' instead." + if not dirExists"dist/nimble": echo "[Error] This script only works for the tarball." else: @@ -11,8 +13,8 @@ else: " dist/nimble/src/nimble.nim" let nimsugExe = "./bin/nimsuggest".toExe - selfExec "c --noNimblePath -p:compiler -o:" & nimsugExe & + selfExec "c --noNimblePath -d:release -p:compiler -o:" & nimsugExe & " dist/nimsuggest/nimsuggest.nim" let nimgrepExe = "./bin/nimgrep".toExe - selfExec "c -o:./bin/nimgrep tools/nimgrep.nim" + selfExec "c -d:release -o:" & nimgrepExe & " tools/nimgrep.nim" diff --git a/koch.nim b/koch.nim index 2f936fc4e..d8004b3e6 100644 --- a/koch.nim +++ b/koch.nim @@ -15,6 +15,11 @@ when defined(gcc) and defined(windows): else: {.link: "icons/koch_icon.o".} +when defined(amd64) and defined(windows) and defined(vcc): + {.link: "icons/koch-amd64-windows-vcc.res" .} +when defined(i386) and defined(windows) and defined(vcc): + {.link: "icons/koch-i386-windows-vcc.res" .} + import os, strutils, parseopt, osproc, streams @@ -35,7 +40,6 @@ Options: --help, -h shows this help and quits Possible Commands: boot [options] bootstraps with given command line options - finish setup PATH and check for a valid GCC installation distrohelper [bindir] helper for distro packagers geninstall generate ./install.sh; Unix only! testinstall test tar.xz package; Unix only! Only for devs! @@ -50,6 +54,9 @@ Possible Commands: tests [options] run the testsuite temp options creates a temporary compiler for testing winrelease creates a release (for coredevs only) + nimble builds the Nimble tool + tools builds Nim related tools + pushcsource push generated C sources to its repo! Only for devs! Boot options: -d:release produce a release version of the compiler -d:tinyc include the Tiny C backend (not supported on Windows) @@ -89,6 +96,9 @@ proc exec(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) putEnv("PATH", prevPath) +proc nimexec(cmd: string) = + exec findNim() & " " & cmd + proc execCleanPath(cmd: string, additionalPath = ""; errorcode: int = QuitFailure) = # simulate a poor man's virtual environment @@ -145,10 +155,13 @@ const compileNimInst = "tools/niminst/niminst" proc csource(args: string) = - exec("$4 cc $1 -r $3 --var:version=$2 --var:mingw=none csource --main:compiler/nim.nim compiler/installer.ini $1" % - [args, VersionAsString, compileNimInst, findNim()]) + nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " & + "--main:compiler/nim.nim compiler/installer.ini $1") % + [args, VersionAsString, compileNimInst]) proc bundleNimbleSrc() = + ## bunldeNimbleSrc() bundles a specific Nimble commit with the tarball. We + ## always bundle the latest official release. if dirExists("dist/nimble/.git"): exec("git --git-dir dist/nimble/.git pull") else: @@ -161,41 +174,77 @@ proc bundleNimbleExe() = bundleNimbleSrc() # now compile Nimble and copy it to $nim/bin for the installer.ini # to pick it up: - exec(findNim() & " c dist/nimble/src/nimble.nim") + nimexec("c dist/nimble/src/nimble.nim") copyExe("dist/nimble/src/nimble".exe, "bin/nimble".exe) -proc bundleNimsuggest(buildExe: bool) = - if dirExists("dist/nimsuggest/.git"): - exec("git --git-dir dist/nimsuggest/.git pull") +proc buildNimble() = + ## buildNimble() builds Nimble for the building via "github". As such, we + ## choose the most recent commit of Nimble too. + var installDir = "dist/nimble" + if dirExists("dist/nimble/.git"): + exec("git --git-dir dist/nimble/.git pull") else: - exec("git clone https://github.com/nim-lang/nimsuggest.git dist/nimsuggest") + # if dist/nimble exist, but is not a git repo, don't mess with it: + if dirExists(installDir): + var id = 0 + while dirExists("dist/nimble" & $id): + inc id + installDir = "dist/nimble" & $id + exec("git clone https://github.com/nim-lang/nimble.git " & installDir) + nimexec("c " & installDir / "src/nimble.nim") + copyExe(installDir / "src/nimble".exe, "bin/nimble".exe) + +proc bundleNimsuggest(buildExe: bool) = if buildExe: - exec(findNim() & " c --noNimblePath -p:compiler dist/nimsuggest/nimsuggest.nim") - copyExe("dist/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) + nimexec("c --noNimblePath -d:release -p:compiler tools/nimsuggest/nimsuggest.nim") + copyExe("tools/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe) + removeFile("tools/nimsuggest/nimsuggest".exe) + +proc bundleWinTools() = + nimexec("c tools/finish.nim") + copyExe("tools/finish".exe, "finish".exe) + removeFile("tools/finish".exe) + nimexec("c -o:bin/vccexe.exe tools/vccenv/vccexe") proc zip(args: string) = bundleNimbleSrc() bundleNimsuggest(false) - exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % - [VersionAsString, compileNimInst, findNim()]) + bundleWinTools() + nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % + [VersionAsString, compileNimInst]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim zip compiler/installer.ini" % ["tools/niminst/niminst".exe, VersionAsString]) proc xz(args: string) = bundleNimbleSrc() bundleNimsuggest(false) - exec("$3 cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % - [VersionAsString, compileNimInst, findNim()]) + nimexec("cc -r $2 --var:version=$1 --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini" % + [VersionAsString, compileNimInst]) exec("$# --var:version=$# --var:mingw=none --main:compiler/nim.nim xz compiler/installer.ini" % ["tools" / "niminst" / "niminst".exe, VersionAsString]) proc buildTool(toolname, args: string) = - exec("$# cc $# $#" % [findNim(), args, toolname]) + nimexec("cc $# $#" % [args, toolname]) copyFile(dest="bin"/ splitFile(toolname).name.exe, source=toolname.exe) +proc buildTools() = + let nimsugExe = "bin/nimsuggest".exe + nimexec "c --noNimblePath -p:compiler -d:release -o:" & nimsugExe & + " tools/nimsuggest/nimsuggest.nim" + + let nimgrepExe = "bin/nimgrep".exe + nimexec "c -o:" & nimgrepExe & " tools/nimgrep.nim" + if dirExists"dist/nimble": + let nimbleExe = "bin/nimble".exe + nimexec "c --noNimblePath -p:compiler -o:" & nimbleExe & + " dist/nimble/src/nimble.nim" + else: + buildNimble() + proc nsis(args: string) = bundleNimbleExe() bundleNimsuggest(true) + bundleWinTools() # make sure we have generated the niminst executables: buildTool("tools/niminst/niminst", args) #buildTool("tools/nimgrep", args) @@ -206,17 +255,21 @@ proc nsis(args: string) = " nsis compiler/installer.ini") % [VersionAsString, $(sizeof(pointer)*8)]) proc geninstall(args="") = - exec("$# cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini $#" % - [findNim(), compileNimInst, VersionAsString, args]) + nimexec("cc -r $# --var:version=$# --var:mingw=none --main:compiler/nim.nim scripts compiler/installer.ini $#" % + [compileNimInst, VersionAsString, args]) + +proc install(args: string) = + geninstall() + exec("sh ./install.sh $#" % args) proc web(args: string) = - exec("$# js tools/dochack/dochack.nim" % findNim()) - exec("$# cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % - [findNim(), args, VersionAsString]) + nimexec("js tools/dochack/dochack.nim") + nimexec("cc -r tools/nimweb.nim $# web/website.ini --putenv:nimversion=$#" % + [args, VersionAsString]) proc website(args: string) = - exec("$# cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" % - [findNim(), args, VersionAsString]) + nimexec("cc -r tools/nimweb.nim $# --website web/website.ini --putenv:nimversion=$#" % + [args, VersionAsString]) proc pdf(args="") = exec("$# cc -r tools/nimweb.nim $# --pdf web/website.ini --putenv:nimversion=$#" % @@ -255,7 +308,7 @@ proc boot(args: string) = var finalDest = "bin" / "nim".exe # default to use the 'c' command: let bootOptions = if args.len == 0 or args.startsWith("-"): "c" else: "" - let smartNimcache = if "release" in args: "rnimcache" else: "dnimcache" + let smartNimcache = if "release" in args: "nimcache/release" else: "nimcache/debug" copyExe(findStartNim(), 0.thVersion) for i in 0..2: @@ -306,7 +359,6 @@ proc removePattern(pattern: string) = removeFile(f) proc clean(args: string) = - if existsFile("koch.dat"): removeFile("koch.dat") removePattern("web/*.html") removePattern("doc/*.html") cleanAux(getCurrentDir()) @@ -320,159 +372,6 @@ proc clean(args: string) = proc winRelease() = exec(r"call ci\nsis_build.bat " & VersionAsString) -# -------------- post unzip steps --------------------------------------------- - -when defined(windows): - import registry - - proc askBool(m: string): bool = - stdout.write m - while true: - let answer = stdin.readLine().normalize - case answer - of "y", "yes": - return true - of "n", "no": - return false - else: - echo "Please type 'y' or 'n'" - - proc askNumber(m: string; a, b: int): int = - stdout.write m - stdout.write " [" & $a & ".." & $b & "] " - while true: - let answer = stdin.readLine() - try: - result = parseInt answer - if result < a or result > b: - raise newException(ValueError, "number out of range") - break - except ValueError: - echo "Please type in a number between ", a, " and ", b - - proc patchConfig(mingw: string) = - const - cfgFile = "config/nim.cfg" - lookFor = """#gcc.path = r"$nim\dist\mingw\bin"""" - replacePattern = """gcc.path = r"$1"""" - try: - let cfg = readFile(cfgFile) - let newCfg = cfg.replace(lookFor, replacePattern % mingw) - if newCfg == cfg: - echo "Could not patch 'config/nim.cfg' [Error]" - echo "Reason: patch substring not found:" - echo lookFor - else: - writeFile(cfgFile, newCfg) - except IOError: - echo "Could not access 'config/nim.cfg' [Error]" - - proc addToPathEnv(e: string) = - let p = getUnicodeValue(r"Environment", "Path", HKEY_CURRENT_USER) - let x = if e.contains(Whitespace): "\"" & e & "\"" else: e - setUnicodeValue(r"Environment", "Path", p & ";" & x, HKEY_CURRENT_USER) - - proc createShortcut(src, dest: string; icon = "") = - var cmd = "bin\\makelink.exe \"" & src & "\" \"\" \"" & dest & - ".lnk\" \"\" 1 \"" & splitFile(src).dir & "\"" - if icon.len != 0: - cmd.add " \"" & icon & "\" 0" - discard execShellCmd(cmd) - - proc createStartMenuEntry() = - let appdata = getEnv("APPDATA") - if appdata.len == 0: return - let dest = appdata & r"\Microsoft\Windows\Start Menu\Programs\Nim-" & - VersionAsString - if dirExists(dest): return - if askBool("Would like to add Nim-" & VersionAsString & - " to your start menu? (y/n) "): - createDir(dest) - createShortcut(getCurrentDir() / "start.bat", dest / "Nim", - getCurrentDir() / r"icons\nim.ico") - if fileExists("doc/overview.html"): - createShortcut(getCurrentDir() / "doc" / "overview.html", - dest / "Overview") - if dirExists(r"dist\aporia-0.4.0"): - createShortcut(getCurrentDir() / r"dist\aporia-0.4.0\bin\aporia.exe", - dest / "Aporia") - - proc checkGccArch(mingw: string): bool = - let gccExe = mingw / r"gcc.exe" - if fileExists(gccExe): - try: - let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut, - poUsePath}) - when hostCPU == "i386": - result = arch.startsWith("i686-") - elif hostCPU == "amd64": - result = arch.startsWith("x86_64-") - else: - {.error: "Unknown CPU for Windows.".} - except OSError, IOError: - result = false - - proc tryDirs(dirs: varargs[string]): string = - let bits = $(sizeof(pointer)*8) - for d in dirs: - if dirExists d: - let x = expandFilename(d / "bin") - if checkGccArch(x): return x - elif dirExists(d & bits): - let x = expandFilename((d & bits) / "bin") - if checkGccArch(x): return x - -proc finish() = - when defined(windows): - let desiredPath = expandFilename(getCurrentDir() / "bin") - let p = getUnicodeValue(r"Environment", "Path", - HKEY_CURRENT_USER) - var alreadyInPath = false - var mingWchoices: seq[string] = @[] - for x in p.split(';'): - let y = expandFilename(if x[0] == '"' and x[^1] == '"': - substr(x, 1, x.len-2) else: x) - if y == desiredPath: alreadyInPath = true - if y.toLowerAscii.contains("mingw"): - if dirExists(y) and checkGccArch(y): - mingWchoices.add y - - if alreadyInPath: - echo "bin/nim.exe is already in your PATH [Skipping]" - else: - if askBool("nim.exe is not in your PATH environment variable.\n" & - " Should it be added permanently? (y/n) "): - addToPathEnv(desiredPath) - if mingWchoices.len == 0: - # No mingw in path, so try a few locations: - let alternative = tryDirs("dist/mingw", "../mingw", r"C:\mingw") - if alternative.len == 0: - echo "No MingW found in PATH and no candidate found " & - " in the standard locations [Error]" - else: - if askBool("Found a MingW directory that is not in your PATH.\n" & - alternative & - "\nShould it be added to your PATH permanently? (y/n) "): - addToPathEnv(alternative) - elif askBool("Do you want to patch Nim's config to use this? (y/n) "): - patchConfig(alternative) - elif mingWchoices.len == 1: - if askBool("MingW installation found at " & mingWchoices[0] & "\n" & - "Do you want to patch Nim's config to use this?\n" & - "(Not required since it's in your PATH!) (y/n) "): - patchConfig(mingWchoices[0]) - else: - echo "Multiple MingW installations found: " - for i in 0..high(mingWchoices): - echo "[", i, "] ", mingWchoices[i] - if askBool("Do you want to patch Nim's config to use one of these? (y/n) "): - let idx = askNumber("Which one do you want to use for Nim? ", - 1, len(mingWchoices)) - patchConfig(mingWchoices[idx-1]) - createStartMenuEntry() - else: - echo("Add ", getCurrentDir(), "/bin to your PATH...") - # -------------- tests -------------------------------------------------------- template `|`(a, b): string = (if a.len > 0: a else: b) @@ -480,11 +379,10 @@ template `|`(a, b): string = (if a.len > 0: a else: b) proc tests(args: string) = # we compile the tester with taintMode:on to have a basic # taint mode test :-) - let nimexe = findNim() - exec nimexe & " cc --taintMode:on tests/testament/tester" + nimexec "cc --taintMode:on tests/testament/tester" # Since tests take a long time (on my machine), and we want to defy Murhpys # law - lets make sure the compiler really is freshly compiled! - exec nimexe & " c --lib:lib -d:release --opt:speed compiler/nim.nim" + nimexec "c --lib:lib -d:release --opt:speed compiler/nim.nim" let tester = quoteShell(getCurrentDir() / "tests/testament/tester".exe) let success = tryExec tester & " " & (args|"all") if not existsEnv("TRAVIS") and not existsEnv("APPVEYOR"): @@ -501,6 +399,38 @@ proc temp(args: string) = copyExe(output, finalDest) if args.len > 0: exec(finalDest & " " & args) +proc copyDir(src, dest: string) = + for kind, path in walkDir(src, relative=true): + case kind + of pcDir: copyDir(dest / path, src / path) + of pcFile: + createDir(dest) + copyFile(src / path, dest / path) + else: discard + +proc pushCsources() = + if not dirExists("../csources/.git"): + quit "[Error] no csources git repository found" + csource("-d:release") + let cwd = getCurrentDir() + try: + copyDir("build/c_code", "../csources/c_code") + copyFile("build/build.sh", "../csources/build.sh") + copyFile("build/build.bat", "../csources/build.bat") + copyFile("build/build64.bat", "../csources/build64.bat") + copyFile("build/makefile", "../csources/makefile") + + setCurrentDir("../csources") + for kind, path in walkDir("c_code"): + if kind == pcDir: + exec("git add " & path / "*.c") + exec("git commit -am \"updated csources to version " & NimVersion & "\"") + exec("git push origin master") + exec("git tag -am \"Version $1\" v$1" % NimVersion) + exec("git push origin v$1" % NimVersion) + finally: + setCurrentDir(cwd) + proc showHelp() = quit(HelpText % [VersionAsString & spaces(44-len(VersionAsString)), CompileDate, CompileTime], QuitSuccess) @@ -512,9 +442,9 @@ of cmdLongOption, cmdShortOption: showHelp() of cmdArgument: case normalize(op.key) of "boot": boot(op.cmdLineRest) - of "finish": finish() of "clean": clean(op.cmdLineRest) of "web": web(op.cmdLineRest) + of "doc", "docs": web("--onlyDocs " & op.cmdLineRest) of "json2": web("--json2 " & op.cmdLineRest) of "website": website(op.cmdLineRest & " --googleAnalytics:UA-48159761-1") of "web0": @@ -527,9 +457,13 @@ of cmdArgument: of "nsis": nsis(op.cmdLineRest) of "geninstall": geninstall(op.cmdLineRest) of "distrohelper": geninstall() + of "install": install(op.cmdLineRest) of "testinstall": testUnixInstall() of "test", "tests": tests(op.cmdLineRest) of "temp": temp(op.cmdLineRest) of "winrelease": winRelease() + of "nimble": buildNimble() + of "tools": buildTools() + of "pushcsource", "pushcsources": pushCsources() else: showHelp() of cmdEnd: showHelp() diff --git a/lib/arch/arch.nim b/lib/arch/arch.nim index c8ae3da1c..0b3df3d3c 100644 --- a/lib/arch/arch.nim +++ b/lib/arch/arch.nim @@ -34,7 +34,7 @@ when defined(amd64): Reg* {.pure.} = enum AX, BX, CX, DX, SI, DI, BP, SP, IP, R8, R9, R10, R11, R12, R13, R14, R15, TOTAL -elif defined(i386): +elif defined(i386) or defined(nimdoc): # identical fastcall calling convention on all x86 OS type JmpBufReg* {.pure.} = enum diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 19452b4a8..b0ef54397 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -78,7 +78,7 @@ type nnkBreakState NimNodeKinds* = set[NimNodeKind] - NimTypeKind* = enum + NimTypeKind* = enum # some types are no longer used, see ast.nim ntyNone, ntyBool, ntyChar, ntyEmpty, ntyArrayConstr, ntyNil, ntyExpr, ntyStmt, ntyTypeDesc, ntyGenericInvocation, ntyGenericBody, ntyGenericInst, @@ -90,9 +90,9 @@ type ntyInt8, ntyInt16, ntyInt32, ntyInt64, ntyFloat, ntyFloat32, ntyFloat64, ntyFloat128, ntyUInt, ntyUInt8, ntyUInt16, ntyUInt32, ntyUInt64, - ntyBigNum, - ntyConst, ntyMutable, ntyVarargs, - ntyIter, + ntyUnused0, ntyUnused1, ntyUnused2, + ntyVarargs, + ntyUnused, ntyError, ntyBuiltinTypeClass, ntyConcept, ntyConceptInst, ntyComposite, ntyAnd, ntyOr, ntyNot @@ -235,7 +235,7 @@ proc getImpl*(s: NimSym): NimNode {.magic: "GetImpl", noSideEffect.} = ## const. discard -proc error*(msg: string) {.magic: "NError", benign.} +proc error*(msg: string, n: NimNode = nil) {.magic: "NError", benign.} ## writes an error message at compile time proc warning*(msg: string) {.magic: "NWarning", benign.} @@ -377,19 +377,19 @@ proc expectKind*(n: NimNode, k: NimNodeKind) {.compileTime.} = ## checks that `n` is of kind `k`. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check the AST that is passed to them. - if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind) + if n.kind != k: error("Expected a node of kind " & $k & ", got " & $n.kind, n) proc expectMinLen*(n: NimNode, min: int) {.compileTime.} = ## checks that `n` has at least `min` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len < min: error("macro expects a node with " & $min & " children") + if n.len < min: error("macro expects a node with " & $min & " children", n) proc expectLen*(n: NimNode, len: int) {.compileTime.} = ## checks that `n` has exactly `len` children. If this is not the case, ## compilation aborts with an error message. This is useful for writing ## macros that check its number of arguments. - if n.len != len: error("macro expects a node with " & $len & " children") + if n.len != len: error("macro expects a node with " & $len & " children", n) proc newTree*(kind: NimNodeKind, children: varargs[NimNode]): NimNode {.compileTime.} = @@ -890,6 +890,30 @@ proc boolVal*(n: NimNode): bool {.compileTime, noSideEffect.} = if n.kind == nnkIntLit: n.intVal != 0 else: n == bindSym"true" # hacky solution for now +macro expandMacros*(body: typed): untyped = + ## Expands one level of macro - useful for debugging. + ## Can be used to inspect what happens when a macro call is expanded, + ## without altering its result. + ## + ## For instance, + ## + ## .. code-block:: nim + ## import future, macros + ## + ## let + ## x = 10 + ## y = 20 + ## expandMacros: + ## dump(x + y) + ## + ## will actually dump `x + y`, but at the same time will print at + ## compile time the expansion of the ``dump`` macro, which in this + ## case is ``debugEcho ["x + y", " = ", x + y]``. + template inner(x: untyped): untyped = x + + result = getAst(inner(body)) + echo result.toStrLit + when not defined(booting): template emit*(e: static[string]): untyped {.deprecated.} = ## accepts a single string argument and treats it as nim code diff --git a/lib/impure/nre.nim b/lib/impure/nre.nim index 557bb0549..626c3fd6b 100644 --- a/lib/impure/nre.nim +++ b/lib/impure/nre.nim @@ -431,8 +431,12 @@ proc initRegex(pattern: string, flags: int, study = true): Regex = raise SyntaxError(msg: $errorMsg, pos: errOffset, pattern: pattern) if study: - # XXX investigate JIT - result.pcreExtra = pcre.study(result.pcreObj, 0x0, addr errorMsg) + var options: cint = 0 + var hasJit: cint + if pcre.config(pcre.CONFIG_JIT, addr hasJit) == 0: + if hasJit == 1'i32: + options = pcre.STUDY_JIT_COMPILE + result.pcreExtra = pcre.study(result.pcreObj, options, addr errorMsg) if errorMsg != nil: raise StudyError(msg: $errorMsg) diff --git a/lib/impure/re.nim b/lib/impure/re.nim index bd86bcdcf..f7f7c5b25 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -87,7 +87,12 @@ proc re*(s: string, flags = {reExtended, reStudy}): Regex = result.h = rawCompile(s, cast[cint](flags - {reStudy})) if reStudy in flags: var msg: cstring - result.e = pcre.study(result.h, 0, addr msg) + var options: cint = 0 + var hasJit: cint + if pcre.config(pcre.CONFIG_JIT, addr hasJit) == 0: + if hasJit == 1'i32: + options = pcre.STUDY_JIT_COMPILE + result.e = pcre.study(result.h, options, addr msg) if not isNil(msg): raiseInvalidRegex($msg) proc matchOrFind(s: string, pattern: Regex, matches: var openArray[string], @@ -214,7 +219,7 @@ proc find*(s: string, pattern: Regex, start = 0): int = var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData - res = pcre.exec(pattern.h, nil, s, len(s).cint, start.cint, 0'i32, + res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, start.cint, 0'i32, cast[ptr cint](rawMatches), 3) if res < 0'i32: return res return rawMatches[0] diff --git a/lib/js/jsconsole.nim b/lib/js/jsconsole.nim new file mode 100644 index 000000000..d9ced95f0 --- /dev/null +++ b/lib/js/jsconsole.nim @@ -0,0 +1,44 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Wrapper for the `console` object for the `JavaScript backend +## <backends.html#the-javascript-target>`_. + +when not defined(js) and not defined(Nimdoc): + {.error: "This module only works on the JavaScript platform".} + +import macros + +type Console* {.importc.} = ref object of RootObj + +proc convertToConsoleLoggable*[T](v: T): RootRef {.importcpp: "#".} +template convertToConsoleLoggable*(v: string): RootRef = cast[RootRef](cstring(v)) + +proc logImpl(console: Console) {.importcpp: "log", varargs.} +proc debugImpl(console: Console) {.importcpp: "debug", varargs.} +proc infoImpl(console: Console) {.importcpp: "info", varargs.} +proc errorImpl(console: Console) {.importcpp: "error", varargs.} + +proc makeConsoleCall(console: NimNode, procName: NimNode, args: NimNode): NimNode = + result = newCall(procName, console) + for c in args: result.add(c) + +macro log*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = + makeConsoleCall(console, bindSym "logImpl", args) + +macro debug*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = + makeConsoleCall(console, bindSym "debugImpl", args) + +macro info*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = + makeConsoleCall(console, bindSym "infoImpl", args) + +macro error*(console: Console, args: varargs[RootRef, convertToConsoleLoggable]): untyped = + makeConsoleCall(console, bindSym "errorImpl", args) + +var console* {.importc, nodecl.}: Console \ No newline at end of file diff --git a/lib/nimbase.h b/lib/nimbase.h index 52de60969..818bff462 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -459,3 +459,7 @@ typedef int Nim_and_C_compiler_disagree_on_target_architecture[sizeof(NI) == siz #elif defined(__FreeBSD__) # include <sys/types.h> #endif + +/* Compile with -d:checkAbi and a sufficiently C11:ish compiler to enable */ +#define NIM_CHECK_SIZE(typ, sz) \ + _Static_assert(sizeof(typ) == sz, "Nim & C disagree on type size") diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 9de25f82b..06b90768c 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -242,14 +242,17 @@ proc nimNextToken(g: var GeneralTokenizer) = inc(pos) case g.buf[pos] of 'b', 'B': + g.kind = gtBinNumber inc(pos) while g.buf[pos] in binChars: inc(pos) pos = nimNumberPostfix(g, pos) of 'x', 'X': + g.kind = gtHexNumber inc(pos) while g.buf[pos] in hexChars: inc(pos) pos = nimNumberPostfix(g, pos) of 'o', 'O': + g.kind = gtOctNumber inc(pos) while g.buf[pos] in octChars: inc(pos) pos = nimNumberPostfix(g, pos) @@ -700,8 +703,8 @@ proc yamlNextToken(g: var GeneralTokenizer) = g.state = gtLongStringLit elif g.state == gtLongStringLit: # beware, this is the only token where we actually have to parse - # indentation. - + # indentation. + g.kind = gtLongStringLit # first, we have to find the parent indentation of the block scalar, so that # we know when to stop @@ -738,20 +741,20 @@ proc yamlNextToken(g: var GeneralTokenizer) = # because lookbehind was at newline char when calculating indentation, we're # off by one. fix that. top level's parent will have indentation of -1. let parentIndentation = indentation - 1 - + # find first content while g.buf[pos] in {' ', '\x0A', '\x0D'}: if g.buf[pos] == ' ': inc(indentation) else: indentation = 0 inc(pos) var minIndentation = indentation - + # for stupid edge cases, we must check whether an explicit indentation depth # is given at the header. while g.buf[headerStart] in {'>', '|', '+', '-'}: inc(headerStart) if g.buf[headerStart] in {'0'..'9'}: minIndentation = min(minIndentation, ord(g.buf[headerStart]) - ord('0')) - + # process content lines while indentation > parentIndentation and g.buf[pos] != '\0': if (indentation < minIndentation and g.buf[pos] == '#') or @@ -766,7 +769,7 @@ proc yamlNextToken(g: var GeneralTokenizer) = if g.buf[pos] == ' ': inc(indentation) else: indentation = 0 inc(pos) - + g.state = gtOther elif g.state == gtOther: # gtOther means 'inside YAML document' diff --git a/lib/posix/kqueue.nim b/lib/posix/kqueue.nim index 5c67d621e..19d661490 100644 --- a/lib/posix/kqueue.nim +++ b/lib/posix/kqueue.nim @@ -123,7 +123,7 @@ when defined(macosx) or defined(freebsd): NOTE_USECONDS* = 0x00000004'u32 ## data is microseconds NOTE_NSECONDS* = 0x00000008'u32 ## data is nanoseconds else: - # NetBSD and OpenBSD doesnt support NOTE_{TIME} constants, but + # NetBSD and OpenBSD doesn't support NOTE_{TIME} constants, but # support EVFILT_TIMER with granularity of milliseconds. const NOTE_MSECONDS* = 0x00000000'u32 diff --git a/lib/pure/asyncdispatch.nim b/lib/pure/asyncdispatch.nim index 8336da1fb..01088c2e7 100644 --- a/lib/pure/asyncdispatch.nim +++ b/lib/pure/asyncdispatch.nim @@ -11,7 +11,7 @@ include "system/inclrtl" import os, oids, tables, strutils, times, heapqueue -import nativesockets, net, queues +import nativesockets, net, deques export Port, SocketFlag @@ -132,7 +132,7 @@ export Port, SocketFlag ## # Handle exception ## ## Unfortunately the semantics of the try statement may not always be correct, -## and occassionally the compilation may fail altogether. +## and occasionally the compilation may fail altogether. ## As such it is better to use the former style when possible. ## ## Discarding futures @@ -164,7 +164,7 @@ include includes/asyncfutures type PDispatcherBase = ref object of RootRef timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] - callbacks: Queue[proc ()] + callbacks: Deque[proc ()] proc processTimers(p: PDispatcherBase) {.inline.} = while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: @@ -172,7 +172,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} = proc processPendingCallbacks(p: PDispatcherBase) = while p.callbacks.len > 0: - var cb = p.callbacks.dequeue() + var cb = p.callbacks.popFirst() cb() proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = @@ -230,7 +230,7 @@ when defined(windows) or defined(nimdoc): result.ioPort = createIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 1) result.handles = initSet[AsyncFD]() result.timers.newHeapQueue() - result.callbacks = initQueue[proc ()](64) + result.callbacks = initDeque[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -987,7 +987,7 @@ else: new result result.selector = newSelector() result.timers.newHeapQueue() - result.callbacks = initQueue[proc ()](64) + result.callbacks = initDeque[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -1043,6 +1043,26 @@ else: p.selector[fd.SocketHandle].data.PData.writeCBs.add(cb) update(fd, p.selector[fd.SocketHandle].events + {EvWrite}) + template processCallbacks(callbacks: expr) = + # Callback may add items to ``callbacks`` which causes issues if + # we are iterating over it at the same time. We therefore + # make a copy to iterate over. + let currentCBs = callbacks + callbacks = @[] + # Using another sequence because callbacks themselves can add + # other callbacks. + var newCBs: seq[Callback] = @[] + for cb in currentCBs: + if newCBs.len > 0: + # A callback has already returned with EAGAIN, don't call any + # others until next `poll`. + newCBs.add(cb) + else: + if not cb(data.fd): + # Callback wants to be called again. + newCBs.add(cb) + callbacks = newCBs & callbacks + proc poll*(timeout = 500) = let p = getGlobalDispatcher() @@ -1055,24 +1075,11 @@ else: # so that exceptions can be raised from `send(...)` and # `recv(...)` routines. - if EvRead in info.events: - # Callback may add items to ``data.readCBs`` which causes issues if - # we are iterating over ``data.readCBs`` at the same time. We therefore - # make a copy to iterate over. - let currentCBs = data.readCBs - data.readCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.readCBs.add(cb) - - if EvWrite in info.events: - let currentCBs = data.writeCBs - data.writeCBs = @[] - for cb in currentCBs: - if not cb(data.fd): - # Callback wants to be called again. - data.writeCBs.add(cb) + if EvRead in info.events or info.events == {EvError}: + processCallbacks(data.readCBs) + + if EvWrite in info.events or info.events == {EvError}: + processCallbacks(data.writeCBs) if info.key in p.selector: var newEvents: set[Event] @@ -1410,7 +1417,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async, deprecated.} = proc callSoon*(cbproc: proc ()) = ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. - getGlobalDispatcher().callbacks.enqueue(cbproc) + getGlobalDispatcher().callbacks.addLast(cbproc) proc runForever*() = ## Begins a never ending global dispatcher poll loop. diff --git a/lib/pure/asyncfile.nim b/lib/pure/asyncfile.nim index ffe6a391e..0241e4796 100644 --- a/lib/pure/asyncfile.nim +++ b/lib/pure/asyncfile.nim @@ -118,8 +118,8 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = ## Read ``size`` bytes from the specified file asynchronously starting at ## the current position of the file pointer. ## - ## If the file pointer is past the end of the file then an empty string is - ## returned. + ## If the file pointer is past the end of the file then zero is returned + ## and no bytes are read into ``buf`` var retFuture = newFuture[int]("asyncfile.readBuffer") when defined(windows) or defined(nimdoc): @@ -149,7 +149,11 @@ proc readBuffer*(f: AsyncFile, buf: pointer, size: int): Future[int] = let err = osLastError() if err.int32 != ERROR_IO_PENDING: GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + if err.int32 == ERROR_HANDLE_EOF: + # This happens in Windows Server 2003 + retFuture.complete(0) + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) else: # Request completed immediately. var bytesRead: DWord @@ -233,7 +237,12 @@ proc read*(f: AsyncFile, size: int): Future[string] = dealloc buffer buffer = nil GC_unref(ol) - retFuture.fail(newException(OSError, osErrorMsg(err))) + + if err.int32 == ERROR_HANDLE_EOF: + # This happens in Windows Server 2003 + retFuture.complete("") + else: + retFuture.fail(newException(OSError, osErrorMsg(err))) else: # Request completed immediately. var bytesRead: DWord diff --git a/lib/pure/asynchttpserver.nim b/lib/pure/asynchttpserver.nim index a658097f9..68da5f7c9 100644 --- a/lib/pure/asynchttpserver.nim +++ b/lib/pure/asynchttpserver.nim @@ -81,6 +81,17 @@ proc respond*(req: Request, code: HttpCode, content: string, ## content. ## ## This procedure will **not** close the client socket. + ## + ## Examples + ## -------- + ## .. code-block::nim + ## proc handler(req: Request) {.async.} = + ## if req.url.path == "/hello-world": + ## let msg = %* {"message": "Hello World"} + ## let headers = newHttpHeaders([("Content-Type","application/json")]) + ## await req.respond(Http200, $msg, headers) + ## else: + ## await req.respond(Http404, "Not Found") var msg = "HTTP/1.1 " & $code & "\c\L" if headers != nil: @@ -122,14 +133,21 @@ proc processClient(client: AsyncSocket, address: string, assert client != nil request.client = client - # First line - GET /path HTTP/1.1 - lineFut.mget().setLen(0) - lineFut.clean() - await client.recvLineInto(lineFut) # TODO: Timeouts. - if lineFut.mget == "": - client.close() - return + # We should skip at least one empty line before the request + # https://tools.ietf.org/html/rfc7230#section-3.5 + for i in 0..1: + lineFut.mget().setLen(0) + lineFut.clean() + await client.recvLineInto(lineFut) # TODO: Timeouts. + if lineFut.mget == "": + client.close() + return + + if lineFut.mget != "\c\L": + break + + # First line - GET /path HTTP/1.1 var i = 0 for linePart in lineFut.mget.split(' '): case i diff --git a/lib/pure/collections/deques.nim b/lib/pure/collections/deques.nim new file mode 100644 index 000000000..c25429778 --- /dev/null +++ b/lib/pure/collections/deques.nim @@ -0,0 +1,266 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Implementation of a `deque`:idx: (double-ended queue). +## The underlying implementation uses a ``seq``. +## +## None of the procs that get an individual value from the deque can be used +## on an empty deque. +## If compiled with `boundChecks` option, those procs will raise an `IndexError` +## on such access. This should not be relied upon, as `-d:release` will +## disable those checks and may return garbage or crash the program. +## +## As such, a check to see if the deque is empty is needed before any +## access, unless your program logic guarantees it indirectly. +## +## .. code-block:: Nim +## proc foo(a, b: Positive) = # assume random positive values for `a` and `b` +## var deq = initDeque[int]() # initializes the object +## for i in 1 ..< a: deq.addLast i # populates the deque +## +## if b < deq.len: # checking before indexed access +## echo "The element at index position ", b, " is ", deq[b] +## +## # The following two lines don't need any checking on access due to the +## # logic of the program, but that would not be the case if `a` could be 0. +## assert deq.peekFirst == 1 +## assert deq.peekLast == a +## +## while deq.len > 0: # checking if the deque is empty +## echo deq.removeLast() +## +## Note: For inter thread communication use +## a `Channel <channels.html>`_ instead. + +import math + +type + Deque*[T] = object + ## A double-ended queue backed with a ringed seq buffer. + data: seq[T] + head, tail, count, mask: int + +proc initDeque*[T](initialSize: int = 4): Deque[T] = + ## Create a new deque. + ## Optionally, the initial capacity can be reserved via `initialSize` as a + ## performance optimization. The length of a newly created deque will still + ## be 0. + ## + ## `initialSize` needs to be a power of two. If you need to accept runtime + ## values for this you could use the ``nextPowerOfTwo`` proc from the + ## `math <math.html>`_ module. + assert isPowerOfTwo(initialSize) + result.mask = initialSize-1 + newSeq(result.data, initialSize) + +proc len*[T](deq: Deque[T]): int {.inline.} = + ## Return the number of elements of `deq`. + result = deq.count + +template emptyCheck(deq) = + # Bounds check for the regular deque access. + when compileOption("boundChecks"): + if unlikely(deq.count < 1): + raise newException(IndexError, "Empty deque.") + +template xBoundsCheck(deq, i) = + # Bounds check for the array like accesses. + when compileOption("boundChecks"): # d:release should disable this. + if unlikely(i >= deq.count): # x < deq.low is taken care by the Natural parameter + raise newException(IndexError, + "Out of bounds: " & $i & " > " & $(deq.count - 1)) + +proc `[]`*[T](deq: Deque[T], i: Natural) : T {.inline.} = + ## Access the i-th element of `deq` by order from first to last. + ## deq[0] is the first, deq[^1] is the last. + xBoundsCheck(deq, i) + return deq.data[(deq.first + i) and deq.mask] + +proc `[]`*[T](deq: var Deque[T], i: Natural): var T {.inline.} = + ## Access the i-th element of `deq` and returns a mutable + ## reference to it. + xBoundsCheck(deq, i) + return deq.data[(deq.head + i) and deq.mask] + +proc `[]=`* [T] (deq: var Deque[T], i: Natural, val : T) {.inline.} = + ## Change the i-th element of `deq`. + xBoundsCheck(deq, i) + deq.data[(deq.head + i) and deq.mask] = val + +iterator items*[T](deq: Deque[T]): T = + ## Yield every element of `deq`. + var i = deq.head + for c in 0 ..< deq.count: + yield deq.data[i] + i = (i + 1) and deq.mask + +iterator mitems*[T](deq: var Deque[T]): var T = + ## Yield every element of `deq`. + var i = deq.head + for c in 0 ..< deq.count: + yield deq.data[i] + i = (i + 1) and deq.mask + +iterator pairs*[T](deq: Deque[T]): tuple[key: int, val: T] = + ## Yield every (position, value) of `deq`. + var i = deq.head + for c in 0 ..< deq.count: + yield (c, deq.data[i]) + i = (i + 1) and deq.mask + +proc contains*[T](deq: Deque[T], item: T): bool {.inline.} = + ## Return true if `item` is in `deq` or false if not found. Usually used + ## via the ``in`` operator. It is the equivalent of ``deq.find(item) >= 0``. + ## + ## .. code-block:: Nim + ## if x in q: + ## assert q.contains x + for e in deq: + if e == item: return true + return false + +proc expandIfNeeded[T](deq: var Deque[T]) = + var cap = deq.mask + 1 + if unlikely(deq.count >= cap): + var n = newSeq[T](cap * 2) + for i, x in deq: # don't use copyMem because the GC and because it's slower. + shallowCopy(n[i], x) + shallowCopy(deq.data, n) + deq.mask = cap * 2 - 1 + deq.tail = deq.count + deq.head = 0 + +proc addFirst*[T](deq: var Deque[T], item: T) = + ## Add an `item` to the beginning of the `deq`. + expandIfNeeded(deq) + inc deq.count + deq.head = (deq.head - 1) and deq.mask + deq.data[deq.head] = item + +proc addLast*[T](deq: var Deque[T], item: T) = + ## Add an `item` to the end of the `deq`. + expandIfNeeded(deq) + inc deq.count + deq.data[deq.tail] = item + deq.tail = (deq.tail + 1) and deq.mask + +proc peekFirst*[T](deq: Deque[T]): T {.inline.}= + ## Returns the first element of `deq`, but does not remove it from the deque. + emptyCheck(deq) + result = deq.data[deq.head] + +proc peekLast*[T](deq: Deque[T]): T {.inline.} = + ## Returns the last element of `deq`, but does not remove it from the deque. + emptyCheck(deq) + result = deq.data[(deq.tail - 1) and deq.mask] + +proc default[T](t: typedesc[T]): T {.inline.} = discard +proc popFirst*[T](deq: var Deque[T]): T {.inline, discardable.} = + ## Remove and returns the first element of the `deq`. + emptyCheck(deq) + dec deq.count + result = deq.data[deq.head] + deq.data[deq.head] = default(type(result)) + deq.head = (deq.head + 1) and deq.mask + +proc popLast*[T](deq: var Deque[T]): T {.inline, discardable.} = + ## Remove and returns the last element of the `deq`. + emptyCheck(deq) + dec deq.count + deq.tail = (deq.tail - 1) and deq.mask + result = deq.data[deq.tail] + deq.data[deq.tail] = default(type(result)) + +proc `$`*[T](deq: Deque[T]): string = + ## Turn a deque into its string representation. + result = "[" + for x in deq: + if result.len > 1: result.add(", ") + result.add($x) + result.add("]") + +when isMainModule: + var deq = initDeque[int](1) + deq.addLast(4) + deq.addFirst(9) + deq.addFirst(123) + var first = deq.popFirst() + deq.addLast(56) + assert(deq.peekLast() == 56) + deq.addLast(6) + assert(deq.peekLast() == 6) + var second = deq.popFirst() + deq.addLast(789) + assert(deq.peekLast() == 789) + + assert first == 123 + assert second == 9 + assert($deq == "[4, 56, 6, 789]") + + assert deq[0] == deq.peekFirst and deq.peekFirst == 4 + assert deq[^1] == deq.peekLast and deq.peekLast == 789 + deq[0] = 42 + deq[^1] = 7 + + assert 6 in deq and 789 notin deq + assert deq.find(6) >= 0 + assert deq.find(789) < 0 + + for i in -2 .. 10: + if i in deq: + assert deq.contains(i) and deq.find(i) >= 0 + else: + assert(not deq.contains(i) and deq.find(i) < 0) + + when compileOption("boundChecks"): + try: + echo deq[99] + assert false + except IndexError: + discard + + try: + assert deq.len == 4 + for i in 0 ..< 5: deq.popFirst() + assert false + except IndexError: + discard + + # grabs some types of resize error. + deq = initDeque[int]() + for i in 1 .. 4: deq.addLast i + deq.popFirst() + deq.popLast() + for i in 5 .. 8: deq.addFirst i + assert $deq == "[8, 7, 6, 5, 2, 3]" + + # Similar to proc from the documentation example + proc foo(a, b: Positive) = # assume random positive values for `a` and `b`. + var deq = initDeque[int]() + assert deq.len == 0 + for i in 1 .. a: deq.addLast i + + if b < deq.len: # checking before indexed access. + assert deq[b] == b + 1 + + # The following two lines don't need any checking on access due to the logic + # of the program, but that would not be the case if `a` could be 0. + assert deq.peekFirst == 1 + assert deq.peekLast == a + + while deq.len > 0: # checking if the deque is empty + assert deq.popFirst() > 0 + + #foo(0,0) + foo(8,5) + foo(10,9) + foo(1,1) + foo(2,1) + foo(1,5) + foo(3,2) \ No newline at end of file diff --git a/lib/pure/collections/queues.nim b/lib/pure/collections/queues.nim index 399e4d413..e4d7eeef1 100644 --- a/lib/pure/collections/queues.nim +++ b/lib/pure/collections/queues.nim @@ -39,8 +39,10 @@ import math +{.warning: "`queues` module is deprecated - use `deques` instead".} + type - Queue*[T] = object ## A queue. + Queue* {.deprecated.} [T] = object ## A queue. data: seq[T] rd, wr, count, mask: int diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index f458b7636..45a148fbf 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -228,7 +228,7 @@ proc apply*[T](data: var seq[T], op: proc (x: var T) {.closure.}) ## var a = @["1", "2", "3", "4"] ## echo repr(a) ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: var string) = x &= "42") + ## apply(a, proc(x: var string) = x &= "42") ## echo repr(a) ## # --> ["142", "242", "342", "442"] ## @@ -247,7 +247,7 @@ proc apply*[T](data: var seq[T], op: proc (x: T): T {.closure.}) ## var a = @["1", "2", "3", "4"] ## echo repr(a) ## # --> ["1", "2", "3", "4"] - ## map(a, proc(x: string): string = x & "42") + ## apply(a, proc(x: string): string = x & "42") ## echo repr(a) ## # --> ["142", "242", "342", "442"] ## diff --git a/lib/pure/collections/tableimpl.nim b/lib/pure/collections/tableimpl.nim index a3dfd43a1..674fdddd2 100644 --- a/lib/pure/collections/tableimpl.nim +++ b/lib/pure/collections/tableimpl.nim @@ -39,16 +39,22 @@ template rawGetKnownHCImpl() {.dirty.} = h = nextTry(h, maxHash(t)) result = -1 - h # < 0 => MISSING; insert idx = -1 - result -template rawGetImpl() {.dirty.} = +template genHashImpl(key, hc: typed) = hc = hash(key) if hc == 0: # This almost never taken branch should be very predictable. hc = 314159265 # Value doesn't matter; Any non-zero favorite is fine. + +template genHash(key: typed): Hash = + var res: Hash + genHashImpl(key, res) + res + +template rawGetImpl() {.dirty.} = + genHashImpl(key, hc) rawGetKnownHCImpl() template rawGetDeepImpl() {.dirty.} = # Search algo for unconditional add - hc = hash(key) - if hc == 0: - hc = 314159265 + genHashImpl(key, hc) var h: Hash = hc and maxHash(t) while isFilled(t.data[h].hcode): h = nextTry(h, maxHash(t)) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 778ea5ca3..e6e72d9ed 100644 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -224,7 +224,7 @@ template withValue*[A, B](t: var Table[A, B], key: A, iterator allValues*[A, B](t: Table[A, B]; key: A): B = ## iterates over any value in the table `t` that belongs to the given `key`. - var h: Hash = hash(key) and high(t.data) + var h: Hash = genHash(key) and high(t.data) while isFilled(t.data[h].hcode): if t.data[h].key == key: yield t.data[h].val @@ -338,7 +338,7 @@ proc hasKey*[A, B](t: TableRef[A, B], key: A): bool = ## returns true iff `key` is in the table `t`. result = t[].hasKey(key) -template equalsImpl(t) = +template equalsImpl(s, t: typed): typed = if s.counter == t.counter: # different insertion orders mean different 'data' seqs, so we have # to use the slow route here: @@ -348,7 +348,9 @@ template equalsImpl(t) = return true proc `==`*[A, B](s, t: Table[A, B]): bool = - equalsImpl(t) + ## The `==` operator for hash tables. Returns ``true`` iff the content of both + ## tables contains the same key-value pairs. Insert order does not matter. + equalsImpl(s, t) proc indexBy*[A, B, C](collection: A, index: proc(x: B): C): Table[C, B] = ## Index the collection with the proc provided. @@ -436,9 +438,12 @@ proc `$`*[A, B](t: TableRef[A, B]): string = dollarImpl() proc `==`*[A, B](s, t: TableRef[A, B]): bool = + ## The `==` operator for hash tables. Returns ``true`` iff either both tables + ## are ``nil`` or none is ``nil`` and the content of both tables contains the + ## same key-value pairs. Insert order does not matter. if isNil(s): result = isNil(t) elif isNil(t): result = false - else: equalsImpl(t[]) + else: equalsImpl(s[], t[]) proc newTableFrom*[A, B, C](collection: A, index: proc(x: B): C): TableRef[C, B] = ## Index the collection with the proc provided. @@ -464,13 +469,17 @@ proc len*[A, B](t: OrderedTable[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -proc clear*[A, B](t: var OrderedTable[A, B] | OrderedTableRef[A, B]) = +proc clear*[A, B](t: var OrderedTable[A, B]) = ## Resets the table so that it is empty. clearImpl() t.first = -1 t.last = -1 -template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = +proc clear*[A, B](t: var OrderedTableRef[A, B]) = + ## Resets the table so that is is empty. + clear(t[]) + +template forAllOrderedPairs(yieldStmt: untyped): typed {.dirty.} = var h = t.first while h >= 0: var nxt = t.data[h].next @@ -606,6 +615,15 @@ proc `$`*[A, B](t: OrderedTable[A, B]): string = ## The `$` operator for ordered hash tables. dollarImpl() +proc `==`*[A, B](s, t: OrderedTable[A, B]): bool = + ## The `==` operator for ordered hash tables. Returns true iff both the + ## content and the order are equal. + if s.counter == t.counter: + forAllOrderedPairs: + if s.data[h] != t.data[h]: return false + result = true + else: result = false + proc sort*[A, B](t: var OrderedTable[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list @@ -656,13 +674,6 @@ proc len*[A, B](t: OrderedTableRef[A, B]): int {.inline.} = ## returns the number of keys in `t`. result = t.counter -template forAllOrderedPairs(yieldStmt: untyped) {.oldimmediate, dirty.} = - var h = t.first - while h >= 0: - var nxt = t.data[h].next - if isFilled(t.data[h].hcode): yieldStmt - h = nxt - iterator pairs*[A, B](t: OrderedTableRef[A, B]): (A, B) = ## iterates over any (key, value) pair in the table `t` in insertion ## order. @@ -738,7 +749,7 @@ proc newOrderedTable*[A, B](initialSize=64): OrderedTableRef[A, B] = ## values for this you could use the ``nextPowerOfTwo`` proc from the ## `math <math.html>`_ module or the ``rightSize`` proc from this module. new(result) - result[] = initOrderedTable[A, B]() + result[] = initOrderedTable[A, B](initialSize) proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): OrderedTableRef[A, B] = ## creates a new ordered hash table that contains the given `pairs`. @@ -749,6 +760,14 @@ proc `$`*[A, B](t: OrderedTableRef[A, B]): string = ## The `$` operator for ordered hash tables. dollarImpl() +proc `==`*[A, B](s, t: OrderedTableRef[A, B]): bool = + ## The `==` operator for ordered hash tables. Returns true iff either both + ## tables are ``nil`` or none is ``nil`` and the content and the order of + ## both are equal. + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: result = s[] == t[] + proc sort*[A, B](t: OrderedTableRef[A, B], cmp: proc (x,y: (A, B)): int) = ## sorts `t` according to `cmp`. This modifies the internal list @@ -759,20 +778,22 @@ proc sort*[A, B](t: OrderedTableRef[A, B], proc del*[A, B](t: var OrderedTable[A, B], key: A) = ## deletes `key` from ordered hash table `t`. O(n) comlexity. - var prev = -1 - let hc = hash(key) - forAllOrderedPairs: - if t.data[h].hcode == hc: - if t.first == h: - t.first = t.data[h].next + var n: OrderedKeyValuePairSeq[A, B] + newSeq(n, len(t.data)) + var h = t.first + t.first = -1 + t.last = -1 + swap(t.data, n) + let hc = genHash(key) + while h >= 0: + var nxt = n[h].next + if isFilled(n[h].hcode): + if n[h].hcode == hc and n[h].key == key: + dec t.counter else: - t.data[prev].next = t.data[h].next - var zeroValue : type(t.data[h]) - t.data[h] = zeroValue - dec t.counter - break - else: - prev = h + var j = -1 - rawGetKnownHC(t, n[h].key, n[h].hcode) + rawInsert(t, t.data, n[h].key, n[h].val, n[h].hcode, j) + h = nxt proc del*[A, B](t: var OrderedTableRef[A, B], key: A) = ## deletes `key` from ordered hash table `t`. O(n) comlexity. @@ -916,11 +937,17 @@ proc `$`*[A](t: CountTable[A]): string = ## The `$` operator for count tables. dollarImpl() +proc `==`*[A](s, t: CountTable[A]): bool = + ## The `==` operator for count tables. Returns ``true`` iff both tables + ## contain the same keys with the same count. Insert order does not matter. + equalsImpl(s, t) + proc inc*[A](t: var CountTable[A], key: A, val = 1) = ## increments `t[key]` by `val`. var index = rawGet(t, key) if index >= 0: inc(t.data[index].val, val) + if t.data[index].val == 0: dec(t.counter) else: if mustRehash(len(t.data), t.counter): enlarge(t) rawInsert(t, t.data, key, val) @@ -1040,6 +1067,14 @@ proc `$`*[A](t: CountTableRef[A]): string = ## The `$` operator for count tables. dollarImpl() +proc `==`*[A](s, t: CountTableRef[A]): bool = + ## The `==` operator for count tables. Returns ``true`` iff either both tables + ## are ``nil`` or none is ``nil`` and both contain the same keys with the same + ## count. Insert order does not matter. + if isNil(s): result = isNil(t) + elif isNil(t): result = false + else: result = s[] == t[] + proc inc*[A](t: CountTableRef[A], key: A, val = 1) = ## increments `t[key]` by `val`. t[].inc(key, val) @@ -1124,6 +1159,20 @@ when isMainModule: doAssert(prev < i) prev = i + block: # Deletion from OrderedTable should account for collision groups. See issue #5057. + # The bug is reproducible only with exact keys + const key1 = "boy_jackpot.inGamma" + const key2 = "boy_jackpot.outBlack" + + var t = { + key1: 0, + key2: 0 + }.toOrderedTable() + + t.del(key1) + assert(t.len == 1) + assert(key2 in t) + var t1 = initCountTable[string]() t2 = initCountTable[string]() diff --git a/lib/pure/future.nim b/lib/pure/future.nim index 67975cfcb..2a6d29933 100644 --- a/lib/pure/future.nim +++ b/lib/pure/future.nim @@ -177,3 +177,24 @@ macro `[]`*(lc: ListComprehension, comp, typ: untyped): untyped = newIdentNode("@"), newNimNode(nnkBracket))), result)))) + + +macro dump*(x: typed): untyped = + ## Dumps the content of an expression, useful for debugging. + ## It accepts any expression and prints a textual representation + ## of the tree representing the expression - as it would appear in + ## source code - together with the value of the expression. + ## + ## As an example, + ## + ## .. code-block:: nim + ## let + ## x = 10 + ## y = 20 + ## dump(x + y) + ## + ## will print ``x + y = 30``. + let s = x.toStrLit + let r = quote do: + debugEcho `s`, " = ", `x` + return r \ No newline at end of file diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 10a786555..00c622d74 100644 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -494,12 +494,15 @@ proc untilElementEnd(x: var XmlParser, result: XmlNode, else: discard result.addNode(parse(x, errors)) of xmlElementEnd: - if cmpIgnoreCase(x.elemName, result.tag) == 0: - next(x) - else: + if cmpIgnoreCase(x.elemName, result.tag) != 0: #echo "5; expected: ", result.htmltag, " ", x.elemName adderr(expected(x, result)) - # do not skip it here! + # this seems to do better match error corrections in browsers: + while x.kind in {xmlElementEnd, xmlWhitespace}: + if x.kind == xmlElementEnd and cmpIgnoreCase(x.elemName, result.tag) == 0: + break + next(x) + next(x) break of xmlEof: adderr(expected(x, result)) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 4404a9426..ba967b14f 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -18,14 +18,14 @@ ## ## .. code-block:: Nim ## var client = newHttpClient() -## echo(getContent("http://google.com")) +## echo client.getContent("http://google.com") ## ## The same action can also be performed asynchronously, simply use the ## ``AsyncHttpClient``: ## ## .. code-block:: Nim ## var client = newAsyncHttpClient() -## echo(await getContent("http://google.com")) +## echo await client.getContent("http://google.com") ## ## The functionality implemented by ``HttpClient`` and ``AsyncHttpClient`` ## is the same, so you can use whichever one suits you best in the examples @@ -50,6 +50,20 @@ ## ## echo client.postContent("http://validator.w3.org/check", multipart=data) ## +## You can also make post requests with custom headers. +## This example sets ``Content-Type`` to ``application/json`` +## and uses a json object for the body +## +## .. code-block:: Nim +## import httpclient, json +## +## let client = newHttpClient() +## client.headers = newHttpHeaders({ "Content-Type": "application/json" }) +## let body = %*{ +## "data": "some text" +## } +## echo client.request("http://some.api", httpMethod = HttpPost, body = $body) +## ## Progress reporting ## ================== ## @@ -103,7 +117,7 @@ ## only basic authentication is supported at the moment. import net, strutils, uri, parseutils, strtabs, base64, os, mimetypes, - math, random, httpcore, times + math, random, httpcore, times, tables import asyncnet, asyncdispatch import nativesockets @@ -432,7 +446,7 @@ proc request*(url: string, httpMethod: string, extraHeaders = "", # get the socket ready. If we are connecting through a proxy to SSL, - # send the appropiate CONNECT header. If not, simply connect to the proper + # send the appropriate CONNECT header. If not, simply connect to the proper # host (which may still be the proxy, for normal HTTP) if proxy != nil and hostUrl.scheme == "https": when defined(ssl): @@ -736,7 +750,7 @@ proc newHttpClient*(userAgent = defUserAgent, ## ``proxy`` specifies an HTTP proxy to use for this HTTP client's ## connections. ## - ## ``timeout`` specifies the number of miliseconds to allow before a + ## ``timeout`` specifies the number of milliseconds to allow before a ## ``TimeoutError`` is raised. new result result.headers = newHttpHeaders() @@ -944,8 +958,10 @@ proc parseResponse(client: HttpClient | AsyncHttpClient, proc newConnection(client: HttpClient | AsyncHttpClient, url: Uri) {.multisync.} = if client.currentURL.hostname != url.hostname or - client.currentURL.scheme != url.scheme: - if client.connected: client.close() + client.currentURL.scheme != url.scheme or + client.currentURL.port != url.port: + if client.connected: + client.close() when client is HttpClient: client.socket = newSocket() @@ -973,8 +989,20 @@ proc newConnection(client: HttpClient | AsyncHttpClient, client.currentURL = url client.connected = true +proc override(fallback, override: HttpHeaders): HttpHeaders = + # Right-biased map union for `HttpHeaders` + if override.isNil: + return fallback + + result = newHttpHeaders() + # Copy by value + result.table[] = fallback.table[] + for k, vs in override.table: + result[k] = vs + proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod: string, body = ""): Future[Response] {.multisync.} = + httpMethod: string, body = "", + headers: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the custom method string specified by ``httpMethod``. ## @@ -1007,13 +1035,15 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, else: await newConnection(client, connectionUrl) - if not client.headers.hasKey("user-agent") and client.userAgent != "": - client.headers["User-Agent"] = client.userAgent + let effectiveHeaders = client.headers.override(headers) + + if not effectiveHeaders.hasKey("user-agent") and client.userAgent != "": + effectiveHeaders["User-Agent"] = client.userAgent - var headers = generateHeaders(requestUrl, httpMethod, - client.headers, body, client.proxy) + var headersString = generateHeaders(requestUrl, httpMethod, + effectiveHeaders, body, client.proxy) - await client.socket.send(headers) + await client.socket.send(headersString) if body != "": await client.socket.send(body) @@ -1024,7 +1054,8 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, client.proxy = savedProxy proc request*(client: HttpClient | AsyncHttpClient, url: string, - httpMethod = HttpGET, body = ""): Future[Response] {.multisync.} = + httpMethod = HttpGET, body = "", + headers: HttpHeaders = nil): Future[Response] {.multisync.} = ## Connects to the hostname specified by the URL and performs a request ## using the method specified. ## @@ -1034,7 +1065,8 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string, ## ## When a request is made to a different hostname, the current connection will ## be closed. - result = await request(client, url, $httpMethod, body) + result = await request(client, url, $httpMethod, body, + headers = headers) proc get*(client: HttpClient | AsyncHttpClient, url: string): Future[Response] {.multisync.} = @@ -1081,18 +1113,22 @@ proc post*(client: HttpClient | AsyncHttpClient, url: string, body = "", else: x var xb = mpBody.withNewLine() & body + + var headers = newHttpHeaders() if multipart != nil: - client.headers["Content-Type"] = mpHeader.split(": ")[1] - client.headers["Content-Length"] = $len(xb) + headers["Content-Type"] = mpHeader.split(": ")[1] + headers["Content-Length"] = $len(xb) - result = await client.request(url, HttpPOST, xb) + result = await client.request(url, HttpPOST, xb, + headers = headers) # Handle redirects. var lastURL = url for i in 1..client.maxRedirects: if result.status.redirection(): let redirectTo = getNewLocation(lastURL, result.headers) var meth = if result.status != "307": HttpGet else: HttpPost - result = await client.request(redirectTo, meth, xb) + result = await client.request(redirectTo, meth, xb, + headers = headers) lastURL = redirectTo proc postContent*(client: HttpClient | AsyncHttpClient, url: string, diff --git a/lib/pure/includes/asyncfutures.nim b/lib/pure/includes/asyncfutures.nim index d78464a91..029c5f157 100644 --- a/lib/pure/includes/asyncfutures.nim +++ b/lib/pure/includes/asyncfutures.nim @@ -246,6 +246,7 @@ proc `or`*[T, Y](fut1: Future[T], fut2: Future[Y]): Future[void] = proc all*[T](futs: varargs[Future[T]]): auto = ## Returns a future which will complete once ## all futures in ``futs`` complete. + ## If the argument is empty, the returned future completes immediately. ## ## If the awaited futures are not ``Future[void]``, the returned future ## will hold the values of all awaited futures in a sequence. @@ -262,13 +263,16 @@ proc all*[T](futs: varargs[Future[T]]): auto = for fut in futs: fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - inc(completedFutures) + inc(completedFutures) + if not retFuture.finished: + if f.failed: + retFuture.fail(f.error) + else: + if completedFutures == totalFutures: + retFuture.complete() - if completedFutures == totalFutures: - retFuture.complete() + if totalFutures == 0: + retFuture.complete() return retFuture @@ -281,15 +285,19 @@ proc all*[T](futs: varargs[Future[T]]): auto = for i, fut in futs: proc setCallback(i: int) = fut.callback = proc(f: Future[T]) = - if f.failed: - retFuture.fail(f.error) - elif not retFuture.finished: - retValues[i] = f.read() - inc(completedFutures) + inc(completedFutures) + if not retFuture.finished: + if f.failed: + retFuture.fail(f.error) + else: + retValues[i] = f.read() - if completedFutures == len(retValues): - retFuture.complete(retValues) + if completedFutures == len(retValues): + retFuture.complete(retValues) setCallback(i) + if retValues.len == 0: + retFuture.complete(retValues) + return retFuture diff --git a/lib/pure/ioselectors.nim b/lib/pure/ioselectors.nim index adb3497ac..744bdbaa1 100644 --- a/lib/pure/ioselectors.nim +++ b/lib/pure/ioselectors.nim @@ -52,13 +52,13 @@ when defined(nimdoc): Vnode, ## BSD specific file change happens User, ## User event is raised Error, ## Error happens while waiting, for descriptor - VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occured) - VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occured) + VnodeWrite, ## NOTE_WRITE (BSD specific, write to file occurred) + VnodeDelete, ## NOTE_DELETE (BSD specific, unlink of file occurred) VnodeExtend, ## NOTE_EXTEND (BSD specific, file extended) VnodeAttrib, ## NOTE_ATTRIB (BSD specific, file attributes changed) VnodeLink, ## NOTE_LINK (BSD specific, file link count changed) VnodeRename, ## NOTE_RENAME (BSD specific, file renamed) - VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occured) + VnodeRevoke ## NOTE_REVOKE (BSD specific, file revoke occurred) ReadyKey*[T] = object ## An object which holds result for descriptor @@ -140,7 +140,7 @@ when defined(nimdoc): proc flush*[T](s: Selector[T]) = ## Flushes all changes was made to kernel pool/queue. - ## This function is usefull only for BSD and MacOS, because + ## This function is useful only for BSD and MacOS, because ## kqueue supports bulk changes to be made. ## On Linux/Windows and other Posix compatible operation systems, ## ``flush`` is alias for `discard`. diff --git a/lib/pure/ioselects/ioselectors_kqueue.nim b/lib/pure/ioselects/ioselectors_kqueue.nim index cdaeeae26..3c0cf4e90 100644 --- a/lib/pure/ioselects/ioselectors_kqueue.nim +++ b/lib/pure/ioselects/ioselectors_kqueue.nim @@ -26,8 +26,8 @@ when defined(macosx) or defined(freebsd): const MAX_DESCRIPTORS_ID = 29 # KERN_MAXFILESPERPROC (MacOS) else: const MAX_DESCRIPTORS_ID = 27 # KERN_MAXFILESPERPROC (FreeBSD) - proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, - newp: pointer, newplen: int): cint + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize, + newp: pointer, newplen: csize): cint {.importc: "sysctl",header: """#include <sys/types.h> #include <sys/sysctl.h>"""} elif defined(netbsd) or defined(openbsd): @@ -35,8 +35,8 @@ elif defined(netbsd) or defined(openbsd): # KERN_MAXFILES, because KERN_MAXFILES is always bigger, # than KERN_MAXFILESPERPROC. const MAX_DESCRIPTORS_ID = 7 # KERN_MAXFILES - proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr int, - newp: pointer, newplen: int): cint + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize, + newp: pointer, newplen: csize): cint {.importc: "sysctl",header: """#include <sys/param.h> #include <sys/sysctl.h>"""} @@ -72,7 +72,7 @@ type SelectEvent* = ptr SelectEventImpl proc newSelector*[T](): Selector[T] = var maxFD = 0.cint - var size = sizeof(cint) + var size = csize(sizeof(cint)) var namearr = [1.cint, MAX_DESCRIPTORS_ID.cint] # Obtain maximum number of file descriptors for process if sysctl(addr(namearr[0]), 2, cast[pointer](addr maxFD), addr size, diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 0b7908c02..5fff7352f 100644 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -954,9 +954,11 @@ proc newIndent(curr, indent: int, ml: bool): int = proc nl(s: var string, ml: bool) = if ml: s.add("\n") -proc escapeJson*(s: string): string = +proc escapeJson*(s: string; result: var string) = ## Converts a string `s` to its JSON representation. - result = newStringOfCap(s.len + s.len shr 3) + ## Appends to ``result``. + const + HexChars = "0123456789ABCDEF" result.add("\"") for x in runes(s): var r = int(x) @@ -967,10 +969,19 @@ proc escapeJson*(s: string): string = of '\\': result.add("\\\\") else: result.add(c) else: - result.add("\\u") - result.add(toHex(r, 4)) + # toHex inlined for more speed (saves stupid string allocations): + result.add("\\u0000") + let start = result.len - 4 + for j in countdown(3, 0): + result[j+start] = HexChars[r and 0xF] + r = r shr 4 result.add("\"") +proc escapeJson*(s: string): string = + ## Converts a string `s` to its JSON representation. + result = newStringOfCap(s.len + s.len shr 3) + escapeJson(s, result) + proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = case node.kind @@ -988,7 +999,7 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, inc i # Need to indent more than { result.indent(newIndent(currIndent, indent, ml)) - result.add(escapeJson(key)) + escapeJson(key, result) result.add(": ") toPretty(result, val, indent, ml, false, newIndent(currIndent, indent, ml)) @@ -999,16 +1010,19 @@ proc toPretty(result: var string, node: JsonNode, indent = 2, ml = true, result.add("{}") of JString: if lstArr: result.indent(currIndent) - result.add(escapeJson(node.str)) + escapeJson(node.str, result) of JInt: if lstArr: result.indent(currIndent) - result.add($node.num) + when defined(js): result.add($node.num) + else: result.add(node.num) of JFloat: if lstArr: result.indent(currIndent) - result.add($node.fnum) + # Fixme: implement new system.add ops for the JS target + when defined(js): result.add($node.fnum) + else: result.add(node.fnum) of JBool: if lstArr: result.indent(currIndent) - result.add($node.bval) + result.add(if node.bval: "true" else: "false") of JArray: if lstArr: result.indent(currIndent) if len(node.elems) != 0: @@ -1057,16 +1071,18 @@ proc toUgly*(result: var string, node: JsonNode) = for key, value in pairs(node.fields): if comma: result.add "," else: comma = true - result.add key.escapeJson() + key.escapeJson(result) result.add ":" result.toUgly value result.add "}" of JString: - result.add node.str.escapeJson() + node.str.escapeJson(result) of JInt: - result.add($node.num) + when defined(js): result.add($node.num) + else: result.add(node.num) of JFloat: - result.add($node.fnum) + when defined(js): result.add($node.fnum) + else: result.add(node.fnum) of JBool: result.add(if node.bval: "true" else: "false") of JNull: @@ -1394,4 +1410,6 @@ when isMainModule: var parsed2 = parseFile("tests/testdata/jsontest2.json") doAssert(parsed2{"repository", "description"}.str=="IRC Library for Haskell", "Couldn't fetch via multiply nested key using {}") + doAssert escapeJson("\10FoobarÄ") == "\"\\u000AFoobar\\u00C4\"" + echo("Tests succeeded!") diff --git a/lib/pure/logging.nim b/lib/pure/logging.nim index b23b1e5bb..5544a4b3f 100644 --- a/lib/pure/logging.nim +++ b/lib/pure/logging.nim @@ -172,18 +172,26 @@ when not defined(js): var (path, name, _) = splitFile(getAppFilename()) result = changeFileExt(path / name, "log") + proc newFileLogger*(file: File, + levelThreshold = lvlAll, + fmtStr = defaultFmtStr): FileLogger = + ## Creates a new file logger. This logger logs to ``file``. + new(result) + result.file = file + result.levelThreshold = levelThreshold + result.fmtStr = fmtStr + proc newFileLogger*(filename = defaultFilename(), mode: FileMode = fmAppend, levelThreshold = lvlAll, fmtStr = defaultFmtStr, bufSize: int = -1): FileLogger = - ## Creates a new file logger. This logger logs to a file. + ## Creates a new file logger. This logger logs to a file, specified + ## by ``fileName``. ## Use ``bufSize`` as size of the output buffer when writing the file ## (-1: use system defaults, 0: unbuffered, >0: fixed buffer size). - new(result) - result.levelThreshold = levelThreshold - result.file = open(filename, mode, bufSize = bufSize) - result.fmtStr = fmtStr + let file = open(filename, mode, bufSize = bufSize) + newFileLogger(file, levelThreshold, fmtStr) # ------ diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index 36e6cf52f..c4c731acf 100644 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -9,6 +9,7 @@ ## This module contains procs for `serialization`:idx: and `deseralization`:idx: ## of arbitrary Nim data structures. The serialization format uses `JSON`:idx:. +## Warning: The serialization format could change in future! ## ## **Restriction**: For objects their type is **not** serialized. This means ## essentially that it does not work if the object has some other runtime @@ -29,6 +30,12 @@ ## a = b ## echo($$a[]) # produces "{}", not "{f: 0}" ## +## # unmarshal +## let c = to[B]("""{"f": 2}""") +## +## # marshal +## let s = $$c + ## **Note**: The ``to`` and ``$$`` operations are available at compile-time! import streams, typeinfo, json, intsets, tables, unicode diff --git a/lib/pure/net.nim b/lib/pure/net.nim index d4f239c49..863a8a6f4 100644 --- a/lib/pure/net.nim +++ b/lib/pure/net.nim @@ -173,7 +173,7 @@ type proc socketError*(socket: Socket, err: int = -1, async = false, - lastError = (-1).OSErrorCode): void + lastError = (-1).OSErrorCode): void {.gcsafe.} proc isDisconnectionError*(flags: set[SocketFlag], lastError: OSErrorCode): bool = @@ -1250,7 +1250,7 @@ proc IPv6_loopback*(): IpAddress = address_v6: [0'u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) proc `==`*(lhs, rhs: IpAddress): bool = - ## Compares two IpAddresses for Equality. Returns two if the addresses are equal + ## Compares two IpAddresses for Equality. Returns true if the addresses are equal if lhs.family != rhs.family: return false if lhs.family == IpAddressFamily.IPv4: for i in low(lhs.address_v4) .. high(lhs.address_v4): diff --git a/lib/pure/nimtracker.nim b/lib/pure/nimtracker.nim new file mode 100644 index 000000000..52fa9da77 --- /dev/null +++ b/lib/pure/nimtracker.nim @@ -0,0 +1,80 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Memory tracking support for Nim. + +when not defined(memTracker) and not isMainModule: + {.error: "Memory tracking support is turned off!".} + +{.push memtracker: off.} +# we import the low level wrapper and are careful not to use Nim's +# memory manager for anything here. +import sqlite3 + +var + dbHandle: PSqlite3 + insertStmt: Pstmt + +template sbind(x: int; value) = + when value is cstring: + let ret = insertStmt.bindText(x, value, value.len.int32, SQLITE_TRANSIENT) + if ret != SQLITE_OK: + quit "could not bind value" + else: + let ret = insertStmt.bindInt64(x, value) + if ret != SQLITE_OK: + quit "could not bind value" + +when defined(memTracker): + proc logEntries(log: TrackLog) {.nimcall, locks: 0, tags: [].} = + for i in 0..log.count-1: + var success = false + let e = log.data[i] + discard sqlite3.reset(insertStmt) + discard clearBindings(insertStmt) + sbind 1, e.op + sbind(2, cast[int](e.address)) + sbind 3, e.size + sbind 4, e.file + sbind 5, e.line + if step(insertStmt) == SQLITE_DONE: + success = true + if not success: + quit "could not write to database!" + +proc execQuery(q: string) = + var s: Pstmt + if prepare_v2(dbHandle, q, q.len.int32, s, nil) == SQLITE_OK: + discard step(s) + if finalize(s) != SQLITE_OK: + quit "could not finalize " & $sqlite3.errmsg(dbHandle) + else: + quit "could not prepare statement " & $sqlite3.errmsg(dbHandle) + +proc setupDb() = + execQuery """create table if not exists tracking( + id integer primary key, + op varchar not null, + address integer not null, + size integer not null, + file varchar not null, + line integer not null + )""" + execQuery "delete from tracking" + +if sqlite3.open("memtrack.db", dbHandle) == SQLITE_OK: + setupDb() + const query = "INSERT INTO tracking(op, address, size, file, line) values (?, ?, ?, ?, ?)" + if prepare_v2(dbHandle, query, + query.len, insertStmt, nil) == SQLITE_OK: + when defined(memTracker): + setTrackLogger logEntries + else: + quit "could not prepare statement B " & $sqlite3.errmsg(dbHandle) +{.pop.} diff --git a/lib/pure/os.nim b/lib/pure/os.nim index cdbe170cc..f077e798a 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -50,6 +50,8 @@ proc c_getenv(env: cstring): cstring {. importc: "getenv", header: "<stdlib.h>".} proc c_putenv(env: cstring): cint {. importc: "putenv", header: "<stdlib.h>".} +proc c_free(p: pointer) {. + importc: "free", header: "<stdlib.h>".} var errno {.importc, header: "<errno.h>".}: cint @@ -303,24 +305,47 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} = proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} = ## Returns the `current working directory`:idx:. - const bufsize = 512 # should be enough when defined(windows): + var bufsize = MAX_PATH.int32 when useWinUnicode: var res = newWideCString("", bufsize) - var L = getCurrentDirectoryW(bufsize, res) - if L == 0'i32: raiseOSError(osLastError()) - result = res$L + while true: + var L = getCurrentDirectoryW(bufsize, res) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break else: result = newString(bufsize) - var L = getCurrentDirectoryA(bufsize, result) - if L == 0'i32: raiseOSError(osLastError()) - setLen(result, L) + while true: + var L = getCurrentDirectoryA(bufsize, result) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + result = newString(L) + bufsize = L + else: + setLen(result, L) + break else: + var bufsize = 1024 # should be enough result = newString(bufsize) - if getcwd(result, bufsize) != nil: - setLen(result, c_strlen(result)) - else: - raiseOSError(osLastError()) + while true: + if getcwd(result, bufsize) != nil: + setLen(result, c_strlen(result)) + break + else: + let err = osLastError() + if err.int32 == ERANGE: + bufsize = bufsize shl 1 + doAssert(bufsize >= 0) + result = newString(bufsize) + else: + raiseOSError(osLastError()) proc setCurrentDir*(newDir: string) {.inline, tags: [].} = ## Sets the `current working directory`:idx:; `OSError` is raised if @@ -336,28 +361,45 @@ proc setCurrentDir*(newDir: string) {.inline, tags: [].} = proc expandFilename*(filename: string): string {.rtl, extern: "nos$1", tags: [ReadDirEffect].} = - ## Returns the full (`absolute`:idx:) path of the file `filename`, raises OSError in case of an error. + ## Returns the full (`absolute`:idx:) path of the file `filename`, + ## raises OSError in case of an error. when defined(windows): - const bufsize = 3072'i32 + var bufsize = MAX_PATH.int32 when useWinUnicode: - var unused: WideCString - var res = newWideCString("", bufsize div 2) - var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) - if L <= 0'i32 or L >= bufsize: - raiseOSError(osLastError()) - result = res$L + var unused: WideCString = nil + var res = newWideCString("", bufsize) + while true: + var L = getFullPathNameW(newWideCString(filename), bufsize, res, unused) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + res = newWideCString("", L) + bufsize = L + else: + result = res$L + break else: - var unused: cstring + var unused: cstring = nil result = newString(bufsize) - var L = getFullPathNameA(filename, bufsize, result, unused) - if L <= 0'i32 or L >= bufsize: raiseOSError(osLastError()) - setLen(result, L) + while true: + var L = getFullPathNameA(filename, bufsize, result, unused) + if L == 0'i32: + raiseOSError(osLastError()) + elif L > bufsize: + result = newString(L) + bufsize = L + else: + setLen(result, L) + break else: - # careful, realpath needs to take an allocated buffer according to Posix: - result = newString(pathMax) - var r = realpath(filename, result) - if r.isNil: raiseOSError(osLastError()) - setLen(result, c_strlen(result)) + # according to Posix we don't need to allocate space for result pathname. + # But we need to free return value with free(3). + var r = realpath(filename, nil) + if r.isNil: + raiseOSError(osLastError()) + else: + result = $r + c_free(cast[pointer](r)) when defined(Windows): proc openHandle(path: string, followSymlink=true): Handle = @@ -571,6 +613,7 @@ proc copyFile*(source, dest: string) {.rtl, extern: "nos$1", if bytesread != bufSize: break dealloc(buf) close(s) + flushFile(d) close(d) proc moveFile*(source, dest: string) {.rtl, extern: "nos$1", @@ -1009,27 +1052,61 @@ proc removeDir*(dir: string) {.rtl, extern: "nos$1", tags: [ of pcDir: removeDir(path) rawRemoveDir(dir) -proc rawCreateDir(dir: string) = +proc rawCreateDir(dir: string): bool = + # Try to create one directory (not the whole path). + # returns `true` for success, `false` if the path has previously existed + # + # This is a thin wrapper over mkDir (or alternatives on other systems), + # so in case of a pre-existing path we don't check that it is a directory. when defined(solaris): - if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST and errno != ENOSYS: + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno in {EEXIST, ENOSYS}: + result = false + else: raiseOSError(osLastError()) elif defined(unix): - if mkdir(dir, 0o777) != 0'i32 and errno != EEXIST: + let res = mkdir(dir, 0o777) + if res == 0'i32: + result = true + elif errno == EEXIST: + result = false + else: + echo res raiseOSError(osLastError()) else: when useWinUnicode: wrapUnary(res, createDirectoryW, dir) else: - var res = createDirectoryA(dir) - if res == 0'i32 and getLastError() != 183'i32: + let res = createDirectoryA(dir) + + if res != 0'i32: + result = true + elif getLastError() == 183'i32: + result = false + else: raiseOSError(osLastError()) -proc createDir*(dir: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = +proc existsOrCreateDir*(dir: string): bool = + ## Check if a `directory`:idx: `dir` exists, and create it otherwise. + ## + ## Does not create parent directories (fails if parent does not exist). + ## Returns `true` if the directory already exists, and `false` + ## otherwise. + result = not rawCreateDir(dir) + if result: + # path already exists - need to check that it is indeed a directory + if not existsDir(dir): + raise newException(IOError, "Failed to create the directory") + +proc createDir*(dir: string) {.rtl, extern: "nos$1", + tags: [WriteDirEffect, ReadDirEffect].} = ## Creates the `directory`:idx: `dir`. ## ## The directory may contain several subdirectories that do not exist yet. ## The full path is created. If this fails, `OSError` is raised. It does **not** - ## fail if the path already exists because for most usages this does not + ## fail if the directory already exists because for most usages this does not ## indicate an error. var omitNext = false when doslike: @@ -1039,8 +1116,12 @@ proc createDir*(dir: string) {.rtl, extern: "nos$1", tags: [WriteDirEffect].} = if omitNext: omitNext = false else: - rawCreateDir(substr(dir, 0, i-1)) - rawCreateDir(dir) + discard existsOrCreateDir(substr(dir, 0, i-1)) + + # The loop does not create the dir itself if it doesn't end in separator + if dir.len > 0 and not omitNext and + dir[^1] notin {DirSep, AltSep}: + discard existsOrCreateDir(dir) proc copyDir*(source, dest: string) {.rtl, extern: "nos$1", tags: [WriteIOEffect, ReadIOEffect], benign.} = @@ -1382,6 +1463,35 @@ when declared(paramCount) or defined(nimdoc): for i in 1..paramCount(): result.add(paramStr(i)) +when defined(freebsd): + proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize, + newp: pointer, newplen: csize): cint + {.importc: "sysctl",header: """#include <sys/types.h> + #include <sys/sysctl.h>"""} + const + CTL_KERN = 1 + KERN_PROC = 14 + KERN_PROC_PATHNAME = 12 + MAX_PATH = 1024 + + proc getApplFreebsd(): string = + var pathLength = csize(MAX_PATH) + result = newString(pathLength) + var req = [CTL_KERN.cint, KERN_PROC.cint, KERN_PROC_PATHNAME.cint, -1.cint] + while true: + let res = sysctl(addr req[0], 4, cast[pointer](addr result[0]), + addr pathLength, nil, 0) + if res < 0: + let err = osLastError() + if err.int32 == ENOMEM: + result = newString(pathLength) + else: + result.setLen(0) # error! + break + else: + result.setLen(pathLength) + break + when defined(linux) or defined(solaris) or defined(bsd) or defined(aix): proc getApplAux(procPath: string): string = result = newString(256) @@ -1426,16 +1536,34 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = # Solaris: # /proc/<pid>/object/a.out (filename only) # /proc/<pid>/path/a.out (complete pathname) - # FreeBSD: /proc/<pid>/file when defined(windows): + var bufsize = int32(MAX_PATH) when useWinUnicode: - var buf = newWideCString("", 256) - var len = getModuleFileNameW(0, buf, 256) - result = buf$len + var buf = newWideCString("", bufsize) + while true: + var L = getModuleFileNameW(0, buf, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + buf = newWideCString("", L) + bufsize = L + else: + result = buf$L + break else: - result = newString(256) - var len = getModuleFileNameA(0, result, 256) - setlen(result, int(len)) + result = newString(bufsize) + while true: + var L = getModuleFileNameA(0, result, bufsize) + if L == 0'i32: + result = "" # error! + break + elif L > bufsize: + result = newString(L) + bufsize = L + else: + setLen(result, L) + break elif defined(macosx): var size: cuint32 getExecPath1(nil, size) @@ -1450,7 +1578,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [ReadIOEffect].} = elif defined(solaris): result = getApplAux("/proc/" & $getpid() & "/path/a.out") elif defined(freebsd): - result = getApplAux("/proc/" & $getpid() & "/file") + result = getApplFreebsd() # little heuristic that may work on other POSIX-like systems: if result.len == 0: result = getApplHeuristic() diff --git a/lib/pure/ospaths.nim b/lib/pure/ospaths.nim index 56671ee62..3d3a105f0 100644 --- a/lib/pure/ospaths.nim +++ b/lib/pure/ospaths.nim @@ -434,8 +434,8 @@ when not declared(getEnv) or defined(nimscript): ## On Windows, network paths are considered absolute too. when doslike: var len = len(path) - result = (len > 1 and path[0] in {'/', '\\'}) or - (len > 2 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') + result = (len > 0 and path[0] in {'/', '\\'}) or + (len > 1 and path[0] in {'a'..'z', 'A'..'Z'} and path[1] == ':') elif defined(macos): result = path.len > 0 and path[0] != ':' elif defined(RISCOS): diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index 6c2debb1b..76bd2dfe1 100644 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -774,7 +774,10 @@ elif not defined(useNimRtl): data.workingDir = workingDir when useProcessAuxSpawn: + var currentDir = getCurrentDir() pid = startProcessAuxSpawn(data) + if workingDir.len > 0: + setCurrentDir(currentDir) else: pid = startProcessAuxFork(data) @@ -835,19 +838,22 @@ elif not defined(useNimRtl): chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2) var res: cint - # FIXME: chdir is global to process if data.workingDir.len > 0: setCurrentDir($data.workingDir) var pid: Pid + var err: OSErrorCode if data.optionPoUsePath: res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + if res != 0'i32: err = osLastError() else: res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv) + if res != 0'i32: err = osLastError() discard posix_spawn_file_actions_destroy(fops) discard posix_spawnattr_destroy(attr) - chck res + if res != 0'i32: raiseOSError(err) + return pid else: proc startProcessAuxFork(data: StartProcessData): Pid = diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index c648b0703..47670cc19 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -512,10 +512,16 @@ proc writeConfig*(dict: Config, filename: string) = kv = key if value != "": ## If the key is not empty if not allCharsInSet(value, SymChars): - kv.add(segmentChar) - kv.add("\"") - kv.add(replace(value)) - kv.add("\"") + if find(value, '"') == -1: + kv.add(segmentChar) + kv.add("\"") + kv.add(replace(value)) + kv.add("\"") + else: + kv.add(segmentChar) + kv.add("\"\"\"") + kv.add(replace(value)) + kv.add("\"\"\"") else: kv.add(segmentChar) kv.add(value) diff --git a/lib/pure/parsexml.nim b/lib/pure/parsexml.nim index d16a55302..978c9c516 100644 --- a/lib/pure/parsexml.nim +++ b/lib/pure/parsexml.nim @@ -142,9 +142,9 @@ proc kind*(my: XmlParser): XmlEventKind {.inline.} = template charData*(my: XmlParser): string = ## returns the character data for the events: ``xmlCharData``, ## ``xmlWhitespace``, ``xmlComment``, ``xmlCData``, ``xmlSpecial`` - ## Raises an assertion in debug mode if ``my.kind`` is not one + ## Raises an assertion in debug mode if ``my.kind`` is not one ## of those events. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind in {xmlCharData, xmlWhitespace, xmlComment, xmlCData, xmlSpecial}) my.a @@ -152,49 +152,49 @@ template charData*(my: XmlParser): string = template elementName*(my: XmlParser): string = ## returns the element name for the events: ``xmlElementStart``, ## ``xmlElementEnd``, ``xmlElementOpen`` - ## Raises an assertion in debug mode if ``my.kind`` is not one + ## Raises an assertion in debug mode if ``my.kind`` is not one ## of those events. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind in {xmlElementStart, xmlElementEnd, xmlElementOpen}) my.a template entityName*(my: XmlParser): string = ## returns the entity name for the event: ``xmlEntity`` - ## Raises an assertion in debug mode if ``my.kind`` is not + ## Raises an assertion in debug mode if ``my.kind`` is not ## ``xmlEntity``. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind == xmlEntity) my.a template attrKey*(my: XmlParser): string = ## returns the attribute key for the event ``xmlAttribute`` - ## Raises an assertion in debug mode if ``my.kind`` is not + ## Raises an assertion in debug mode if ``my.kind`` is not ## ``xmlAttribute``. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.a template attrValue*(my: XmlParser): string = ## returns the attribute value for the event ``xmlAttribute`` - ## Raises an assertion in debug mode if ``my.kind`` is not + ## Raises an assertion in debug mode if ``my.kind`` is not ## ``xmlAttribute``. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind == xmlAttribute) my.b template piName*(my: XmlParser): string = ## returns the processing instruction name for the event ``xmlPI`` - ## Raises an assertion in debug mode if ``my.kind`` is not + ## Raises an assertion in debug mode if ``my.kind`` is not ## ``xmlPI``. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.a template piRest*(my: XmlParser): string = ## returns the rest of the processing instruction for the event ``xmlPI`` - ## Raises an assertion in debug mode if ``my.kind`` is not + ## Raises an assertion in debug mode if ``my.kind`` is not ## ``xmlPI``. In release mode, this will not trigger an error - ## but the value returned will not be valid. + ## but the value returned will not be valid. assert(my.kind == xmlPI) my.b @@ -572,6 +572,10 @@ proc parseAttribute(my: var XmlParser) = inc(pos) else: markError(my, errQuoteExpected) + # error corrections: guess what was meant + while buf[pos] != '>' and buf[pos] > ' ': + add(my.b, buf[pos]) + inc pos my.bufpos = pos parseWhitespace(my, skip=true) @@ -636,12 +640,14 @@ proc rawGetTok(my: var XmlParser) = proc getTok(my: var XmlParser) = while true: + let lastKind = my.kind rawGetTok(my) case my.kind of xmlComment: if my.options.contains(reportComments): break of xmlWhitespace: - if my.options.contains(reportWhitespace): break + if my.options.contains(reportWhitespace) or lastKind in {xmlCharData, xmlComment, xmlEntity}: + break else: break proc next*(my: var XmlParser) = diff --git a/lib/pure/random.nim b/lib/pure/random.nim index 08da771dc..955a70143 100644 --- a/lib/pure/random.nim +++ b/lib/pure/random.nim @@ -122,7 +122,7 @@ when isMainModule: inc occur[x] for i, oc in occur: if oc < 69: - doAssert false, "too few occurances of " & $i + doAssert false, "too few occurrences of " & $i elif oc > 130: - doAssert false, "too many occurances of " & $i + doAssert false, "too many occurrences of " & $i main() diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index bfc32bc71..14877eb4d 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -179,9 +179,10 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar, if s.len() == 0: return false - result = true for c in s: - result = c.isLowerAscii() and result + if not c.isLowerAscii(): + return false + true proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, rtl, extern: "nsuIsUpperAsciiStr".}= @@ -193,9 +194,10 @@ proc isUpperAscii*(s: string): bool {.noSideEffect, procvar, if s.len() == 0: return false - result = true for c in s: - result = c.isUpperAscii() and result + if not c.isUpperAscii(): + return false + true proc toLowerAscii*(c: char): char {.noSideEffect, procvar, rtl, extern: "nsuToLowerAsciiChar".} = @@ -1307,10 +1309,12 @@ proc join*[T: not string](a: openArray[T], sep: string = ""): string {. type SkipTable = array[char, int] +{.push profiler: off.} proc preprocessSub(sub: string, a: var SkipTable) = var m = len(sub) for i in 0..0xff: a[chr(i)] = m+1 for i in 0..m-1: a[sub[i]] = m-i +{.pop.} proc findAux(s, sub: string, start: int, a: SkipTable): int = # Fast "quick search" algorithm: diff --git a/lib/pure/times.nim b/lib/pure/times.nim index b78a2b966..1767a37be 100644 --- a/lib/pure/times.nim +++ b/lib/pure/times.nim @@ -66,12 +66,6 @@ when defined(posix) and not defined(JS): when not defined(freebsd) and not defined(netbsd) and not defined(openbsd): var timezone {.importc, header: "<time.h>".}: int - var - tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] - # we also need tzset() to make sure that tzname is initialized - proc tzset() {.importc, header: "<time.h>".} - # calling tzset() implicitly to initialize tzname data. - tzset() elif defined(windows): import winlean @@ -82,12 +76,10 @@ elif defined(windows): # visual c's c runtime exposes these under a different name var timezone {.importc: "_timezone", header: "<time.h>".}: int - tzname {.importc: "_tzname", header: "<time.h>"}: array[0..1, cstring] else: - type TimeImpl {.importc: "time_t", header: "<time.h>".} = int32 + type TimeImpl {.importc: "time_t", header: "<time.h>".} = int var timezone {.importc, header: "<time.h>".}: int - tzname {.importc, header: "<time.h>" .}: array[0..1, cstring] type Time* = distinct TimeImpl @@ -104,7 +96,7 @@ elif defined(JS): getMinutes: proc (): int {.tags: [], raises: [], benign.} getMonth: proc (): int {.tags: [], raises: [], benign.} getSeconds: proc (): int {.tags: [], raises: [], benign.} - getTime: proc (): int {.tags: [], raises: [], benign.} + getTime: proc (): int {.tags: [], raises: [], noSideEffect, benign.} getTimezoneOffset: proc (): int {.tags: [], raises: [], benign.} getDate: proc (): int {.tags: [], raises: [], benign.} getUTCDate: proc (): int {.tags: [], raises: [], benign.} @@ -152,11 +144,14 @@ type yearday*: range[0..365] ## The number of days since January 1, ## in the range 0 to 365. ## Always 0 if the target is JS. - isDST*: bool ## Determines whether DST is in effect. Always - ## ``False`` if time is UTC. - tzname*: string ## The timezone this time is in. E.g. GMT + isDST*: bool ## Determines whether DST is in effect. + ## Semantically, this adds another negative hour + ## offset to the time in addition to the timezone. timezone*: int ## The offset of the (non-DST) timezone in seconds - ## west of UTC. + ## west of UTC. Note that the sign of this number + ## is the opposite of the one in a formatted + ## timezone string like ``+01:00`` (which would be + ## parsed into the timezone ``-3600``). ## I make some assumptions about the data in here. Either ## everything should be positive or everything negative. Zero is @@ -184,7 +179,8 @@ proc getGMTime*(t: Time): TimeInfo {.tags: [TimeEffect], raises: [], benign.} ## converts the calendar time `t` to broken-down time representation, ## expressed in Coordinated Universal Time (UTC). -proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} +proc timeInfoToTime*(timeInfo: TimeInfo): Time + {.tags: [TimeEffect], benign, deprecated.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -193,7 +189,7 @@ proc timeInfoToTime*(timeInfo: TimeInfo): Time {.tags: [], benign, deprecated.} ## **Warning:** This procedure is deprecated since version 0.14.0. ## Use ``toTime`` instead. -proc toTime*(timeInfo: TimeInfo): Time {.tags: [], benign.} +proc toTime*(timeInfo: TimeInfo): Time {.tags: [TimeEffect], benign.} ## converts a broken-down time structure to ## calendar time representation. The function ignores the specified ## contents of the structure members `weekday` and `yearday` and recomputes @@ -211,36 +207,25 @@ proc fromSeconds*(since1970: int64): Time {.tags: [], raises: [], benign.} = proc toSeconds*(time: Time): float {.tags: [], raises: [], benign.} ## Returns the time in seconds since the unix epoch. -proc `$` *(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} - ## converts a `TimeInfo` object to a string representation. -proc `$` *(time: Time): string {.tags: [], raises: [], benign.} - ## converts a calendar time to a string representation. - proc `-`*(a, b: Time): int64 {. - rtl, extern: "ntDiffTime", tags: [], raises: [], benign.} + rtl, extern: "ntDiffTime", tags: [], raises: [], noSideEffect, benign.} ## computes the difference of two calendar times. Result is in seconds. proc `<`*(a, b: Time): bool {. - rtl, extern: "ntLtTime", tags: [], raises: [].} = + rtl, extern: "ntLtTime", tags: [], raises: [], noSideEffect.} = ## returns true iff ``a < b``, that is iff a happened before b. result = a - b < 0 proc `<=` * (a, b: Time): bool {. - rtl, extern: "ntLeTime", tags: [], raises: [].}= + rtl, extern: "ntLeTime", tags: [], raises: [], noSideEffect.}= ## returns true iff ``a <= b``. result = a - b <= 0 proc `==`*(a, b: Time): bool {. - rtl, extern: "ntEqTime", tags: [], raises: [].} = + rtl, extern: "ntEqTime", tags: [], raises: [], noSideEffect.} = ## returns true if ``a == b``, that is if both times represent the same value result = a - b == 0 -when not defined(JS): - proc getTzname*(): tuple[nonDST, DST: string] {.tags: [TimeEffect], raises: [], - benign.} - ## returns the local timezone; ``nonDST`` is the name of the local non-DST - ## timezone, ``DST`` is the name of the local DST timezone. - proc getTimezone*(): int {.tags: [TimeEffect], raises: [], benign.} ## returns the offset of the local (non-DST) timezone in seconds west of UTC. @@ -266,13 +251,13 @@ proc initInterval*(milliseconds, seconds, minutes, hours, days, months, result.milliseconds = `mod`(milliseconds, 1000) carryO = `div`(milliseconds, 1000) result.seconds = `mod`(carryO + seconds, 60) - carryO = `div`(seconds, 60) + carryO = `div`(carryO + seconds, 60) result.minutes = `mod`(carryO + minutes, 60) - carryO = `div`(minutes, 60) + carryO = `div`(carryO + minutes, 60) result.hours = `mod`(carryO + hours, 24) - carryO = `div`(hours, 24) + carryO = `div`(carryO + hours, 24) result.days = carryO + days - carryO = 0 + result.months = `mod`(months, 12) carryO = `div`(months, 12) result.years = carryO + years @@ -283,27 +268,31 @@ proc `+`*(ti1, ti2: TimeInterval): TimeInterval = result.milliseconds = `mod`(ti1.milliseconds + ti2.milliseconds, 1000) carryO = `div`(ti1.milliseconds + ti2.milliseconds, 1000) result.seconds = `mod`(carryO + ti1.seconds + ti2.seconds, 60) - carryO = `div`(ti1.seconds + ti2.seconds, 60) + carryO = `div`(carryO + ti1.seconds + ti2.seconds, 60) result.minutes = `mod`(carryO + ti1.minutes + ti2.minutes, 60) - carryO = `div`(ti1.minutes + ti2.minutes, 60) + carryO = `div`(carryO + ti1.minutes + ti2.minutes, 60) result.hours = `mod`(carryO + ti1.hours + ti2.hours, 24) - carryO = `div`(ti1.hours + ti2.hours, 24) + carryO = `div`(carryO + ti1.hours + ti2.hours, 24) result.days = carryO + ti1.days + ti2.days - carryO = 0 + result.months = `mod`(ti1.months + ti2.months, 12) carryO = `div`(ti1.months + ti2.months, 12) result.years = carryO + ti1.years + ti2.years +proc `-`*(ti: TimeInterval): TimeInterval = + result = TimeInterval( + milliseconds: -ti.milliseconds, + seconds: -ti.seconds, + minutes: -ti.minutes, + hours: -ti.hours, + days: -ti.days, + months: -ti.months, + years: -ti.years + ) + proc `-`*(ti1, ti2: TimeInterval): TimeInterval = ## Subtracts TimeInterval ``ti1`` from ``ti2``. - result = ti1 - result.milliseconds -= ti2.milliseconds - result.seconds -= ti2.seconds - result.minutes -= ti2.minutes - result.hours -= ti2.hours - result.days -= ti2.days - result.months -= ti2.months - result.years -= ti2.years + result = ti1 + (-ti2) proc isLeapYear*(year: int): bool = ## returns true if ``year`` is a leap year @@ -369,7 +358,7 @@ proc `+`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## very accurate. let t = toSeconds(toTime(a)) let secs = toSeconds(a, interval) - if a.tzname == "UTC": + if a.timezone == 0: result = getGMTime(fromSeconds(t + secs)) else: result = getLocalTime(fromSeconds(t + secs)) @@ -379,17 +368,10 @@ proc `-`*(a: TimeInfo, interval: TimeInterval): TimeInfo = ## ## **Note:** This has been only briefly tested, it is inaccurate especially ## when you subtract so much that you reach the Julian calendar. - let t = toSeconds(toTime(a)) - var intval: TimeInterval - intval.milliseconds = - interval.milliseconds - intval.seconds = - interval.seconds - intval.minutes = - interval.minutes - intval.hours = - interval.hours - intval.days = - interval.days - intval.months = - interval.months - intval.years = - interval.years - let secs = toSeconds(a, intval) - if a.tzname == "UTC": + let + t = toSeconds(toTime(a)) + secs = toSeconds(a, -interval) + if a.timezone == 0: result = getGMTime(fromSeconds(t + secs)) else: result = getLocalTime(fromSeconds(t + secs)) @@ -424,7 +406,8 @@ when not defined(JS): when not defined(JS): # C wrapper: - when defined(freebsd) or defined(netbsd) or defined(openbsd): + when defined(freebsd) or defined(netbsd) or defined(openbsd) or + defined(macosx): type StructTM {.importc: "struct tm", final.} = object second {.importc: "tm_sec".}, @@ -461,12 +444,6 @@ when not defined(JS): importc: "time", header: "<time.h>", tags: [].} proc mktime(t: StructTM): Time {. importc: "mktime", header: "<time.h>", tags: [].} - proc asctime(tblock: StructTM): cstring {. - importc: "asctime", header: "<time.h>", tags: [].} - proc ctime(time: ptr Time): cstring {. - importc: "ctime", header: "<time.h>", tags: [].} - # strftime(s: CString, maxsize: int, fmt: CString, t: tm): int {. - # importc: "strftime", header: "<time.h>".} proc getClock(): Clock {.importc: "clock", header: "<time.h>", tags: [TimeEffect].} proc difftime(a, b: Time): float {.importc: "difftime", header: "<time.h>", tags: [].} @@ -479,46 +456,17 @@ when not defined(JS): const weekDays: array[0..6, WeekDay] = [ dSun, dMon, dTue, dWed, dThu, dFri, dSat] - when defined(freebsd) or defined(netbsd) or defined(openbsd): - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - tzname: if local: - if tm.isdst > 0: - getTzname().DST - else: - getTzname().nonDST - else: - "UTC", - # BSD stores in `gmtoff` offset east of UTC in seconds, - # but posix systems using west of UTC in seconds - timezone: if local: -(tm.gmtoff) else: 0 - ) - else: - TimeInfo(second: int(tm.second), - minute: int(tm.minute), - hour: int(tm.hour), - monthday: int(tm.monthday), - month: Month(tm.month), - year: tm.year + 1900'i32, - weekday: weekDays[int(tm.weekday)], - yearday: int(tm.yearday), - isDST: tm.isdst > 0, - tzname: if local: - if tm.isdst > 0: - getTzname().DST - else: - getTzname().nonDST - else: - "UTC", - timezone: if local: getTimezone() else: 0 - ) + TimeInfo(second: int(tm.second), + minute: int(tm.minute), + hour: int(tm.hour), + monthday: int(tm.monthday), + month: Month(tm.month), + year: tm.year + 1900'i32, + weekday: weekDays[int(tm.weekday)], + yearday: int(tm.yearday), + isDST: tm.isdst > 0, + timezone: if local: getTimezone() else: 0 + ) proc timeInfoToTM(t: TimeInfo): StructTM = @@ -569,29 +517,18 @@ when not defined(JS): proc timeInfoToTime(timeInfo: TimeInfo): Time = var cTimeInfo = timeInfo # for C++ we have to make a copy, # because the header of mktime is broken in my version of libc - return mktime(timeInfoToTM(cTimeInfo)) + result = mktime(timeInfoToTM(cTimeInfo)) + # mktime is defined to interpret the input as local time. As timeInfoToTM + # does ignore the timezone, we need to adjust this here. + result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) proc toTime(timeInfo: TimeInfo): Time = var cTimeInfo = timeInfo # for C++ we have to make a copy, # because the header of mktime is broken in my version of libc - return mktime(timeInfoToTM(cTimeInfo)) - - proc toStringTillNL(p: cstring): string = - result = "" - var i = 0 - while p[i] != '\0' and p[i] != '\10' and p[i] != '\13': - add(result, p[i]) - inc(i) - - proc `$`(timeInfo: TimeInfo): string = - # BUGFIX: asctime returns a newline at the end! - var p = asctime(timeInfoToTM(timeInfo)) - result = toStringTillNL(p) - - proc `$`(time: Time): string = - # BUGFIX: ctime returns a newline at the end! - var a = time - return toStringTillNL(ctime(addr(a))) + result = mktime(timeInfoToTM(cTimeInfo)) + # mktime is defined to interpret the input as local time. As timeInfoToTM + # does ignore the timezone, we need to adjust this here. + result = Time(TimeImpl(result) - getTimezone() + timeInfo.timezone) const epochDiff = 116444736000000000'i64 @@ -605,9 +542,6 @@ when not defined(JS): ## converts a Windows time to a UNIX `Time` (``time_t``) result = Time((t - epochDiff) div rateDiff) - proc getTzname(): tuple[nonDST, DST: string] = - return ($tzname[0], $tzname[1]) - proc getTimezone(): int = when defined(freebsd) or defined(netbsd) or defined(openbsd): var a = timec(nil) @@ -675,26 +609,16 @@ elif defined(JS): result.weekday = weekDays[t.getUTCDay()] result.yearday = 0 - proc timeInfoToTime*(timeInfo: TimeInfo): Time = - result = internGetTime() - result.setSeconds(timeInfo.second) - result.setMinutes(timeInfo.minute) - result.setHours(timeInfo.hour) - result.setMonth(ord(timeInfo.month)) - result.setFullYear(timeInfo.year) - result.setDate(timeInfo.monthday) + proc timeInfoToTime*(timeInfo: TimeInfo): Time = toTime(timeInfo) proc toTime*(timeInfo: TimeInfo): Time = result = internGetTime() - result.setSeconds(timeInfo.second) result.setMinutes(timeInfo.minute) result.setHours(timeInfo.hour) result.setMonth(ord(timeInfo.month)) result.setFullYear(timeInfo.year) result.setDate(timeInfo.monthday) - - proc `$`(timeInfo: TimeInfo): string = return $(toTime(timeInfo)) - proc `$`(time: Time): string = return $time.toLocaleString() + result.setSeconds(timeInfo.second + timeInfo.timezone) proc `-` (a, b: Time): int64 = return a.getTime() - b.getTime() @@ -802,6 +726,13 @@ proc `-`*(t: Time, ti: TimeInterval): Time = ## ``echo getTime() - 1.day`` result = toTime(getLocalTime(t) - ti) +const + secondsInMin = 60 + secondsInHour = 60*60 + secondsInDay = 60*60*24 + minutesInHour = 60 + epochStartYear = 1970 + proc formatToken(info: TimeInfo, token: string, buf: var string) = ## Helper of the format proc to parse individual tokens. ## @@ -891,24 +822,32 @@ proc formatToken(info: TimeInfo, token: string, buf: var string) = if fyear.len != 5: fyear = repeat('0', 5-fyear.len()) & fyear buf.add(fyear) of "z": - let hrs = (info.timezone div 60) div 60 - buf.add($hrs) + let + nonDstTz = info.timezone - int(info.isDst) * secondsInHour + hours = abs(nonDstTz) div secondsInHour + if nonDstTz <= 0: buf.add('+') + else: buf.add('-') + buf.add($hours) of "zz": - let hrs = (info.timezone div 60) div 60 - - buf.add($hrs) - if hrs.abs < 10: - var atIndex = buf.len-(($hrs).len-(if hrs < 0: 1 else: 0)) - buf.insert("0", atIndex) + let + nonDstTz = info.timezone - int(info.isDst) * secondsInHour + hours = abs(nonDstTz) div secondsInHour + if nonDstTz <= 0: buf.add('+') + else: buf.add('-') + if hours < 10: buf.add('0') + buf.add($hours) of "zzz": - let hrs = (info.timezone div 60) div 60 - - buf.add($hrs & ":00") - if hrs.abs < 10: - var atIndex = buf.len-(($hrs & ":00").len-(if hrs < 0: 1 else: 0)) - buf.insert("0", atIndex) - of "ZZZ": - buf.add(info.tzname) + let + nonDstTz = info.timezone - int(info.isDst) * secondsInHour + hours = abs(nonDstTz) div secondsInHour + minutes = (abs(nonDstTz) div secondsInMin) mod minutesInHour + if nonDstTz <= 0: buf.add('+') + else: buf.add('-') + if hours < 10: buf.add('0') + buf.add($hours) + buf.add(':') + if minutes < 10: buf.add('0') + buf.add($minutes) of "": discard else: @@ -945,8 +884,7 @@ proc format*(info: TimeInfo, f: string): string = ## yyyy Displays the year to four digits. ``2012 -> 2012`` ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:00``. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` + ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` ## ========== ================================================================================= ================================================ ## ## Other strings can be inserted by putting them in ``''``. For example @@ -984,6 +922,18 @@ proc format*(info: TimeInfo, f: string): string = inc(i) +proc `$`*(timeInfo: TimeInfo): string {.tags: [], raises: [], benign.} = + ## converts a `TimeInfo` object to a string representation. + ## It uses the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + try: + result = format(timeInfo, "yyyy-MM-dd'T'HH:mm:sszzz") # todo: optimize this + except ValueError: assert false # cannot happen because format string is valid + +proc `$`*(time: Time): string {.tags: [TimeEffect], raises: [], benign.} = + ## converts a `Time` value to a string representation. It will use the local + ## time zone and use the format ``yyyy-MM-dd'T'HH-mm-sszzz``. + $getLocalTime(time) + {.pop.} proc parseToken(info: var TimeInfo; token, value: string; j: var int) = @@ -1052,7 +1002,6 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = of "M": var pd = parseInt(value[j..j+1], sv) info.month = Month(sv-1) - info.monthday = sv j += pd of "MM": var month = value[j..j+1].parseInt() @@ -1141,42 +1090,59 @@ proc parseToken(info: var TimeInfo; token, value: string; j: var int) = info.year = value[j..j+3].parseInt() j += 4 of "z": + info.isDST = false if value[j] == '+': - info.timezone = parseInt($value[j+1]) + info.timezone = 0 - parseInt($value[j+1]) * secondsInHour elif value[j] == '-': - info.timezone = 0-parseInt($value[j+1]) + info.timezone = parseInt($value[j+1]) * secondsInHour + elif value[j] == 'Z': + info.timezone = 0 + j += 1 + return else: raise newException(ValueError, "Couldn't parse timezone offset (z), got: " & value[j]) j += 2 of "zz": + info.isDST = false if value[j] == '+': - info.timezone = value[j+1..j+2].parseInt() + info.timezone = 0 - value[j+1..j+2].parseInt() * secondsInHour elif value[j] == '-': - info.timezone = 0-value[j+1..j+2].parseInt() + info.timezone = value[j+1..j+2].parseInt() * secondsInHour + elif value[j] == 'Z': + info.timezone = 0 + j += 1 + return else: raise newException(ValueError, "Couldn't parse timezone offset (zz), got: " & value[j]) j += 3 of "zzz": - if value[j] == '+': - info.timezone = value[j+1..j+2].parseInt() - elif value[j] == '-': - info.timezone = 0-value[j+1..j+2].parseInt() + info.isDST = false + var factor = 0 + if value[j] == '+': factor = -1 + elif value[j] == '-': factor = 1 + elif value[j] == 'Z': + info.timezone = 0 + j += 1 + return else: raise newException(ValueError, "Couldn't parse timezone offset (zzz), got: " & value[j]) - j += 6 - of "ZZZ": - info.tzname = value[j..j+2].toUpperAscii() - j += 3 + info.timezone = factor * value[j+1..j+2].parseInt() * secondsInHour + j += 4 + info.timezone += factor * value[j..j+1].parseInt() * 60 + j += 2 else: # Ignore the token and move forward in the value string by the same length j += token.len proc parse*(value, layout: string): TimeInfo = - ## This function parses a date/time string using the standard format identifiers (below) - ## The function defaults information not provided in the format string from the running program (timezone, month, year, etc) + ## This function parses a date/time string using the standard format + ## identifiers as listed below. The function defaults information not provided + ## in the format string from the running program (timezone, month, year, etc). + ## Daylight saving time is only set if no timezone is given and the given date + ## lies within the DST period of the current locale. ## ## ========== ================================================================================= ================================================ ## Specifier Description Example @@ -1201,10 +1167,9 @@ proc parse*(value, layout: string): TimeInfo = ## tt Same as above, but ``AM`` and ``PM`` instead of ``A`` and ``P`` respectively. ## yy Displays the year to two digits. ``2012 -> 12`` ## yyyy Displays the year to four digits. ``2012 -> 2012`` - ## z Displays the timezone offset from UTC. ``GMT+7 -> +7``, ``GMT-5 -> -5`` + ## z Displays the timezone offset from UTC. ``Z`` is parsed as ``+0`` ``GMT+7 -> +7``, ``GMT-5 -> -5`` ## zz Same as above but with leading 0. ``GMT+7 -> +07``, ``GMT-5 -> -05`` - ## zzz Same as above but with ``:00``. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` - ## ZZZ Displays the name of the timezone. ``GMT -> GMT``, ``EST -> EST`` + ## zzz Same as above but with ``:mm`` where *mm* represents minutes. ``GMT+7 -> +07:00``, ``GMT-5 -> -05:00`` ## ========== ================================================================================= ================================================ ## ## Other strings can be inserted by putting them in ``''``. For example @@ -1220,6 +1185,8 @@ proc parse*(value, layout: string): TimeInfo = info.hour = 0 info.minute = 0 info.second = 0 + info.isDST = true # using this is flag for checking whether a timezone has \ + # been read (because DST is always false when a tz is parsed) while true: case layout[i] of ' ', '-', '/', ':', '\'', '\0', '(', ')', '[', ']', ',': @@ -1248,12 +1215,19 @@ proc parse*(value, layout: string): TimeInfo = else: parseToken(info, token, value, j) token = "" - # Reset weekday (might not have been provided and the default may be wrong) - # and yearday (is never provided directly and therefore probably wrong) - let processed = getLocalTime(toTime(info)) - info.weekday = processed.weekday - info.yearday = processed.yearday - return info + + if info.isDST: + # means that no timezone has been parsed. In this case, we need to check + # whether the date is within DST of the local time. + let tmp = getLocalTime(toTime(info)) + # correctly set isDST so that the following step works on the correct time + info.isDST = tmp.isDST + + # Correct weekday and yearday; transform timestamp to local time. + # There currently is no way of returning this with the original (parsed) + # timezone while also setting weekday and yearday (we are depending on stdlib + # to provide this calculation). + return getLocalTime(toTime(info)) # Leap year calculations are adapted from: # http://www.codeproject.com/Articles/7358/Ultra-fast-Algorithms-for-Working-with-Leap-Years @@ -1284,12 +1258,6 @@ proc countYearsAndDays*(daySpan: int): tuple[years: int, days: int] = result.years = days div 365 result.days = days mod 365 -const - secondsInMin = 60 - secondsInHour = 60*60 - secondsInDay = 60*60*24 - epochStartYear = 1970 - proc getDayOfWeek*(day, month, year: int): WeekDay = ## Returns the day of the week enum from day, month and year. # Day & month start from one. diff --git a/lib/pure/unittest.nim b/lib/pure/unittest.nim index 0fc2e441e..cdca02ed7 100644 --- a/lib/pure/unittest.nim +++ b/lib/pure/unittest.nim @@ -11,11 +11,21 @@ ## ## This module implements boilerplate to make unit testing easy. ## +## The test status and name is printed after any output or traceback. +## ## Example: ## ## .. code:: nim ## ## suite "description for this stuff": +## echo "suite setup: run once before the tests" +## +## setup: +## echo "run before each test" +## +## teardown: +## echo "run after each test": +## ## test "essential truths": ## # give up and stop if this fails ## require(true) @@ -30,6 +40,13 @@ ## let v = @[1, 2, 3] # you can do initialization here ## expect(IndexError): ## discard v[4] +## +## echo "suite teardown: run once after the tests" +## +## +## Tests can be nested, however failure of a nested test will not mark the +## parent test as failed. Setup and teardown are inherited. Setup can be +## overridden locally. import macros @@ -81,7 +98,7 @@ proc startSuite(name: string) = template rawPrint() = echo("\n[Suite] ", name) when not defined(ECMAScript): if colorOutput: - styledEcho styleBright, fgBlue, "\n[Suite] ", fgWhite, name + styledEcho styleBright, fgBlue, "\n[Suite] ", resetStyle, name else: rawPrint() else: rawPrint() @@ -142,7 +159,7 @@ proc testDone(name: string, s: TestStatus, indent: bool) = of FAILED: fgRed of SKIPPED: fgYellow else: fgWhite - styledEcho styleBright, color, prefix, "[", $s, "] ", fgWhite, name + styledEcho styleBright, color, prefix, "[", $s, "] ", resetStyle, name else: rawPrint() else: @@ -226,10 +243,11 @@ template fail* = checkpoints = @[] template skip* = - ## Makes test to be skipped. Should be used directly + ## Mark the test as skipped. Should be used directly ## in case when it is not possible to perform test ## for reasons depending on outer environment, ## or certain application logic conditions or configurations. + ## The test code is still executed. ## ## .. code-block:: nim ## @@ -250,12 +268,12 @@ macro check*(conditions: untyped): untyped = ## ## import strutils ## - ## check("AKB48".toLower() == "akb48") + ## check("AKB48".toLowerAscii() == "akb48") ## ## let teams = {'A', 'K', 'B', '4', '8'} ## ## check: - ## "AKB48".toLower() == "akb48" + ## "AKB48".toLowerAscii() == "akb48" ## 'C' in teams let checked = callsite()[1] var diff --git a/lib/pure/uri.nim b/lib/pure/uri.nim index f0a3796a5..e7db39c10 100644 --- a/lib/pure/uri.nim +++ b/lib/pure/uri.nim @@ -264,6 +264,11 @@ proc `/`*(x: Uri, path: string): Uri = ## let bar = parseUri("http://example.com/foo/bar/") / "baz" ## assert bar.path == "/foo/bar/baz" result = x + + if result.path.len == 0: + result.path = path + return + if result.path[result.path.len-1] == '/': if path[0] == '/': result.path.add(path[1 .. path.len-1]) @@ -442,3 +447,8 @@ when isMainModule: # bug #3207 block: doAssert parseUri("http://qq/1").combine(parseUri("https://qqq")).`$` == "https://qqq" + + # bug #4959 + block: + let foo = parseUri("http://example.com") / "/baz" + doAssert foo.path == "/baz" diff --git a/lib/pure/xmlparser.nim b/lib/pure/xmlparser.nim index 2a2c3e1dd..22bd259b7 100644 --- a/lib/pure/xmlparser.nim +++ b/lib/pure/xmlparser.nim @@ -28,7 +28,7 @@ proc raiseInvalidXml(errors: seq[string]) = proc addNode(father, son: XmlNode) = if son != nil: add(father, son) -proc parse(x: var XmlParser, errors: var seq[string]): XmlNode +proc parse(x: var XmlParser, errors: var seq[string]): XmlNode {.gcsafe.} proc untilElementEnd(x: var XmlParser, result: XmlNode, errors: var seq[string]) = @@ -164,3 +164,6 @@ when isMainModule: var xml = loadXml(filePath, errors) assert(errors.len == 0, "The file tests/testdata/doc1.xml should be parsed without errors.") + block bug1518: + var err: seq[string] = @[] + assert $parsexml(newStringStream"<tag>One & two</tag>", "temp.xml", err) == "<tag>One & two</tag>" diff --git a/lib/stdlib.nimble b/lib/stdlib.nimble index bd1b78efe..6949bfcc4 100644 --- a/lib/stdlib.nimble +++ b/lib/stdlib.nimble @@ -1,6 +1,6 @@ [Package] name = "stdlib" -version = "0.15.1" +version = "0.15.3" author = "Dominik Picheta" description = "Nim's standard library." license = "MIT" diff --git a/lib/system.nim b/lib/system.nim index 4c8af3d51..69d3db291 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1272,12 +1272,14 @@ const seqShallowFlag = low(int) +{.push profiler: off.} when defined(nimKnowsNimvm): let nimvm* {.magic: "Nimvm".}: bool = false ## may be used only in "when" expression. ## It is true in Nim VM context and false otherwise else: const nimvm*: bool = false +{.pop.} proc compileOption*(option: string): bool {. magic: "CompileOption", noSideEffect.} @@ -1827,7 +1829,7 @@ const NimMinor*: int = 15 ## is the minor number of Nim's version. - NimPatch*: int = 1 + NimPatch*: int = 3 ## is the patch number of Nim's version. NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch @@ -2176,25 +2178,34 @@ proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} = for i in 0..y.len-1: result[i+1] = y[i] -when not defined(nimscript): - when not defined(JS) or defined(nimphp): - proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = - result = cast[pointer](x) +proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = + ## Generic equals operator for sequences: relies on a equals operator for + ## the element type `T`. + when nimvm: + if x.isNil and y.isNil: + return true else: - proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} = - asm """return `x`""" + when not defined(JS) or defined(nimphp): + proc seqToPtr[T](x: seq[T]): pointer {.inline, nosideeffect.} = + result = cast[pointer](x) + else: + proc seqToPtr[T](x: seq[T]): pointer {.asmNoStackFrame, nosideeffect.} = + asm """return `x`""" - proc `==` *[T](x, y: seq[T]): bool {.noSideEffect.} = - ## Generic equals operator for sequences: relies on a equals operator for - ## the element type `T`. if seqToPtr(x) == seqToPtr(y): - result = true - elif seqToPtr(x) == nil or seqToPtr(y) == nil: - result = false - elif x.len == y.len: - for i in 0..x.len-1: - if x[i] != y[i]: return false - result = true + return true + + if x.isNil or y.isNil: + return false + + if x.len != y.len: + return false + + for i in 0..x.len-1: + if x[i] != y[i]: + return false + + return true proc find*[T, S](a: T, item: S): int {.inline.}= ## Returns the first index of `item` in `a` or -1 if not found. This requires @@ -2325,7 +2336,11 @@ proc collectionToString[T: set | seq](x: T, b, e: string): string = var firstElement = true for value in items(x): if not firstElement: result.add(", ") - result.add($value) + when compiles(value.isNil): + if value.isNil: result.add "nil" + else: result.add($value) + else: + result.add($value) firstElement = false result.add(e) @@ -2531,6 +2546,7 @@ when hostOS == "standalone": include "$projectpath/panicoverride" when not declared(sysFatal): + {.push profiler: off.} when hostOS == "standalone": proc sysFatal(exceptn: typedesc, message: string) {.inline.} = panic(message) @@ -2550,6 +2566,7 @@ when not declared(sysFatal): new(e) e.msg = message & arg raise e + {.pop.} proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo", benign.} ## get type information for `x`. Ordinary code should not use this, but @@ -2581,10 +2598,11 @@ else: when not defined(JS): #and not defined(nimscript): {.push stack_trace: off, profiler:off.} - when not defined(nimscript) and not defined(nogc): + when hasAlloc: when not defined(gcStack): proc initGC() - when not defined(boehmgc) and not defined(useMalloc) and not defined(gogc) and not defined(gcStack): + when not defined(boehmgc) and not defined(useMalloc) and + not defined(gogc) and not defined(gcStack): proc initAllocator() {.inline.} proc initStackBottom() {.inline, compilerproc.} = @@ -2602,9 +2620,10 @@ when not defined(JS): #and not defined(nimscript): when declared(setStackBottom): setStackBottom(locals) - when hasAlloc: + {.push profiler: off.} var strDesc = TNimType(size: sizeof(string), kind: tyString, flags: {ntfAcyclic}) + {.pop.} # ----------------- IO Part ------------------------------------------------ @@ -2616,6 +2635,8 @@ when not defined(JS): #and not defined(nimscript): FileMode* = enum ## The file mode when opening a file. fmRead, ## Open the file for read access only. fmWrite, ## Open the file for write access only. + ## If the file does not exist, it will be + ## created. fmReadWrite, ## Open the file for read and write access. ## If the file does not exist, it will be ## created. @@ -2632,10 +2653,15 @@ when not defined(JS): #and not defined(nimscript): include "system/ansi_c" - when not defined(nimscript): - proc cmp(x, y: string): int = + proc cmp(x, y: string): int = + when nimvm: + if x < y: result = -1 + elif x > y: result = 1 + else: result = 0 + else: result = int(c_strcmp(x, y)) + when not defined(nimscript): proc zeroMem(p: pointer, size: Natural) = c_memset(p, 0, size) proc copyMem(dest, source: pointer, size: Natural) = @@ -2645,12 +2671,6 @@ when not defined(JS): #and not defined(nimscript): proc equalMem(a, b: pointer, size: Natural): bool = c_memcmp(a, b, size) == 0 - else: - proc cmp(x, y: string): int = - if x < y: result = -1 - elif x > y: result = 1 - else: result = 0 - when defined(nimscript): proc readFile*(filename: string): string {.tags: [ReadIOEffect], benign.} ## Opens a file named `filename` for reading, calls `readAll @@ -2936,6 +2956,8 @@ when not defined(JS): #and not defined(nimscript): ## lead to the ``raise`` statement. This only works for debug builds. {.push stack_trace: off, profiler:off.} + when defined(memtracker): + include "system/memtracker" when hostOS == "standalone": include "system/embedded" else: @@ -2978,7 +3000,9 @@ when not defined(JS): #and not defined(nimscript): else: result = n.sons[n.len] + {.push profiler:off.} when hasAlloc: include "system/mmdisp" + {.pop.} {.push stack_trace: off, profiler:off.} when hasAlloc: include "system/sysstr" {.pop.} @@ -3676,7 +3700,7 @@ template closureScope*(body: untyped): untyped = when defined(nimconfig): include "system/nimscript" -when defined(windows) and appType == "console" and not defined(nimconfig): +when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage): proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32", importc: "SetConsoleOutputCP".} discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 745bbbf62..3a8e8a1b6 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -15,6 +15,10 @@ include osalloc +template track(op, address, size) = + when defined(memTracker): + memTrackerOp(op, address, size) + # We manage *chunks* of memory. Each chunk is a multiple of the page size. # Each chunk starts at an address that is divisible by the page size. Chunks # that are bigger than ``ChunkOsReturn`` are returned back to the operating @@ -645,6 +649,7 @@ proc alloc(allocator: var MemRegion, size: Natural): pointer = cast[ptr FreeCell](result).zeroField = 1 # mark it as used sysAssert(not isAllocatedPtr(allocator, result), "alloc") result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell)) + track("alloc", result, size) proc alloc0(allocator: var MemRegion, size: Natural): pointer = result = alloc(allocator, size) @@ -658,6 +663,7 @@ proc dealloc(allocator: var MemRegion, p: pointer) = sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc 2") rawDealloc(allocator, x) sysAssert(not isAllocatedPtr(allocator, x), "dealloc 3") + track("dealloc", p, 0) proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer = if newsize > 0: diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index dcf41b67d..d00ab64b1 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -339,6 +339,8 @@ when not defined(noSignalHandler): action("unknown signal\n") # print stack trace and quit + when defined(memtracker): + logPendingOps() when hasSomeStackTrace: GC_disable() var buf = newStringOfCap(2000) diff --git a/lib/system/gc.nim b/lib/system/gc.nim index c8623f2f1..7fb4c7ac7 100644 --- a/lib/system/gc.nim +++ b/lib/system/gc.nim @@ -471,6 +471,7 @@ proc rawNewObj(typ: PNimType, size: int, gch: var GcHeap): pointer = # its refcount is zero, so add it to the ZCT: addNewObjToZCT(res, gch) when logGC: writeCell("new cell", res) + track("rawNewObj", res, size) gcTrace(res, csAllocated) release(gch) when useCellIds: @@ -516,6 +517,7 @@ proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} = res.refcount = rcIncrement # refcount is 1 sysAssert(isAllocatedPtr(gch.region, res), "newObj: 3") when logGC: writeCell("new cell", res) + track("newObjRC1", res, size) gcTrace(res, csAllocated) release(gch) when useCellIds: @@ -558,6 +560,8 @@ proc growObj(old: pointer, newsize: int, gch: var GcHeap): pointer = writeCell("growObj new cell", res) gcTrace(ol, csZctFreed) gcTrace(res, csAllocated) + track("growObj old", ol, 0) + track("growObj new", res, newsize) when reallyDealloc: sysAssert(allocInv(gch.region), "growObj before dealloc") if ol.refcount shr rcShift <=% 1: @@ -601,6 +605,7 @@ proc growObj(old: pointer, newsize: int): pointer {.rtl.} = proc freeCyclicCell(gch: var GcHeap, c: PCell) = prepareDealloc(c) gcTrace(c, csCycFreed) + track("cycle collector dealloc cell", c, 0) when logGC: writeCell("cycle collector dealloc cell", c) when reallyDealloc: sysAssert(allocInv(gch.region), "free cyclic cell") @@ -670,6 +675,7 @@ proc doOperation(p: pointer, op: WalkOp) = gcAssert(c.refcount >=% rcIncrement, "doOperation 2") #c.refcount = c.refcount -% rcIncrement when logGC: writeCell("decref (from doOperation)", c) + track("waZctDecref", p, 0) decRef(c) #if c.refcount <% rcIncrement: addZCT(gch.zct, c) of waPush: @@ -762,6 +768,7 @@ proc collectZCT(gch: var GcHeap): bool = # In any case, it should be removed from the ZCT. But not # freed. **KEEP THIS IN MIND WHEN MAKING THIS INCREMENTAL!** when logGC: writeCell("zct dealloc cell", c) + track("zct dealloc cell", c, 0) gcTrace(c, csZctFreed) # We are about to free the object, call the finalizer BEFORE its # children are deleted as well, because otherwise the finalizer may diff --git a/lib/system/gc_stack.nim b/lib/system/gc_stack.nim index 5f72b8959..3eda08df9 100644 --- a/lib/system/gc_stack.nim +++ b/lib/system/gc_stack.nim @@ -79,6 +79,7 @@ template withRegion*(r: MemRegion; body: untyped) = try: body finally: + r = tlRegion tlRegion = oldRegion template inc(p: pointer, s: int) = @@ -464,4 +465,10 @@ proc getFreeMem(): int = tlRegion.remaining proc getTotalMem(): int = result = tlRegion.totalSize +proc getOccupiedMem*(r: MemRegion): int = + result = r.totalSize - r.remaining +proc getFreeMem*(r: MemRegion): int = r.remaining +proc getTotalMem*(r: MemRegion): int = + result = r.totalSize + proc setStackBottom(theStackBottom: pointer) = discard diff --git a/lib/system/hti.nim b/lib/system/hti.nim index 984f888cb..892a209df 100644 --- a/lib/system/hti.nim +++ b/lib/system/hti.nim @@ -62,7 +62,6 @@ type tyUInt16, tyUInt32, tyUInt64, - tyBigNum, TNimNodeKind = enum nkNone, nkSlot, nkList, nkCase TNimNode {.codegenType.} = object diff --git a/lib/system/memtracker.nim b/lib/system/memtracker.nim new file mode 100644 index 000000000..a9767bbca --- /dev/null +++ b/lib/system/memtracker.nim @@ -0,0 +1,70 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Memory tracking support for Nim. + +when not defined(memTracker): + {.error: "Memory tracking support is turned off! Enable memory tracking by passing `--memtracker:on` to the compiler (see the Nim Compiler User Guide for more options).".} + +when defined(noSignalHandler): + {.error: "Memory tracking works better with the default signal handler.".} + +# We don't want to memtrack the tracking code ... +{.push memtracker: off.} + +type + LogEntry* = object + op*: cstring + address*: pointer + size*: int + file*: cstring + line*: int + TrackLog* = object + count*: int + disabled: bool + data*: array[4000, LogEntry] + TrackLogger* = proc (log: TrackLog) {.nimcall, tags: [], locks: 0.} + +var + gLog*: TrackLog + gLogger*: TrackLogger = proc (log: TrackLog) = discard + +proc setTrackLogger*(logger: TrackLogger) = + gLogger = logger + +proc addEntry(entry: LogEntry) = + if not gLog.disabled: + if gLog.count > high(gLog.data): + gLogger(gLog) + gLog.count = 0 + gLog.data[gLog.count] = entry + inc gLog.count + +proc memTrackerWrite(address: pointer; size: int; file: cstring; line: int) {.compilerProc.} = + addEntry LogEntry(op: "write", address: address, + size: size, file: file, line: line) + +proc memTrackerOp*(op: cstring; address: pointer; size: int) = + addEntry LogEntry(op: op, address: address, size: size, + file: "", line: 0) + +proc memTrackerDisable*() = + gLog.disabled = true + +proc memTrackerEnable*() = + gLog.disabled = false + +proc logPendingOps() {.noconv.} = + # forward declared and called from Nim's signal handler. + gLogger(gLog) + gLog.count = 0 + +addQuitProc logPendingOps + +{.pop.} diff --git a/lib/system/osalloc.nim b/lib/system/osalloc.nim index 316dd74d7..b0639a75a 100644 --- a/lib/system/osalloc.nim +++ b/lib/system/osalloc.nim @@ -81,20 +81,27 @@ elif defined(posix): const PROT_READ = 1 # page can be read PROT_WRITE = 2 # page can be written - MAP_PRIVATE = 2'i32 # Changes are private when defined(macosx) or defined(bsd): const MAP_ANONYMOUS = 0x1000 + const MAP_PRIVATE = 0x02 # Changes are private elif defined(solaris): const MAP_ANONYMOUS = 0x100 + const MAP_PRIVATE = 0x02 # Changes are private + elif defined(linux) and defined(amd64): + # actually, any architecture using asm-generic, but being conservative here, + # some arches like mips and alpha use different values + const MAP_ANONYMOUS = 0x20 + const MAP_PRIVATE = 0x02 # Changes are private else: var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + MAP_PRIVATE {.importc: "MAP_PRIVATE", header: "<sys/mman.h>".}: cint - proc mmap(adr: pointer, len: int, prot, flags, fildes: cint, + proc mmap(adr: pointer, len: csize, prot, flags, fildes: cint, off: int): pointer {.header: "<sys/mman.h>".} - proc munmap(adr: pointer, len: int): cint {.header: "<sys/mman.h>".} + proc munmap(adr: pointer, len: csize): cint {.header: "<sys/mman.h>".} proc osAllocPages(size: int): pointer {.inline.} = result = mmap(nil, size, PROT_READ or PROT_WRITE, diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 3e9657ce0..5c10392f1 100644 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -104,21 +104,22 @@ proc getFileHandle*(f: File): FileHandle = c_fileno(f) proc readLine(f: File, line: var TaintedString): bool = var pos = 0 + var sp: cint = 80 # Use the currently reserved space for a first try - when defined(nimscript): - var space: cint = 80 + if line.string.isNil: + line = TaintedString(newStringOfCap(80)) else: - var space: cint = cint(cast[PGenericSeq](line.string).space) - line.string.setLen(space) - + when not defined(nimscript): + sp = cint(cast[PGenericSeq](line.string).space) + line.string.setLen(sp) while true: # memset to \l so that we can tell how far fgets wrote, even on EOF, where # fgets doesn't append an \l - c_memset(addr line.string[pos], '\l'.ord, space) - if c_fgets(addr line.string[pos], space, f) == nil: + c_memset(addr line.string[pos], '\l'.ord, sp) + if c_fgets(addr line.string[pos], sp, f) == nil: line.string.setLen(0) return false - let m = c_memchr(addr line.string[pos], '\l'.ord, space) + let m = c_memchr(addr line.string[pos], '\l'.ord, sp) if m != nil: # \l found: Could be our own or the one by fgets, in any case, we're done var last = cast[ByteAddress](m) - cast[ByteAddress](addr line.string[0]) @@ -129,17 +130,17 @@ proc readLine(f: File, line: var TaintedString): bool = # \0\l\0 => line ending in a null character. # \0\l\l => last line without newline, null was put there by fgets. elif last > 0 and line.string[last-1] == '\0': - if last < pos + space - 1 and line.string[last+1] != '\0': + if last < pos + sp - 1 and line.string[last+1] != '\0': dec last line.string.setLen(last) return true else: # fgets will have inserted a null byte at the end of the string. - dec space + dec sp # No \l found: Increase buffer and read more - inc pos, space - space = 128 # read in 128 bytes at a time - line.string.setLen(pos+space) + inc pos, sp + sp = 128 # read in 128 bytes at a time + line.string.setLen(pos+sp) proc readLine(f: File): TaintedString = result = TaintedString(newStringOfCap(80)) diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 3a93221e0..11034006a 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -263,27 +263,32 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {. result.len = newLen # --------------- other string routines ---------------------------------- -proc nimIntToStr(x: int): string {.compilerRtl.} = - result = newString(sizeof(x)*4) +proc add*(result: var string; x: int64) = + let base = result.len + setLen(result, base + sizeof(x)*4) var i = 0 var y = x while true: var d = y div 10 - result[i] = chr(abs(int(y - d*10)) + ord('0')) + result[base+i] = chr(abs(int(y - d*10)) + ord('0')) inc(i) y = d if y == 0: break if x < 0: - result[i] = '-' + result[base+i] = '-' inc(i) - setLen(result, i) + setLen(result, base+i) # mirror the string: for j in 0..i div 2 - 1: - swap(result[j], result[i-j-1]) + swap(result[base+j], result[base+i-j-1]) -proc nimFloatToStr(f: float): string {.compilerproc.} = +proc nimIntToStr(x: int): string {.compilerRtl.} = + result = newStringOfCap(sizeof(x)*4) + result.add x + +proc add*(result: var string; x: float) = var buf: array[0..64, char] - var n: int = c_sprintf(buf, "%.16g", f) + var n: int = c_sprintf(buf, "%.16g", x) var hasDot = false for i in 0..n-1: if buf[i] == ',': @@ -298,14 +303,18 @@ proc nimFloatToStr(f: float): string {.compilerproc.} = # On Windows nice numbers like '1.#INF', '-1.#INF' or '1.#NAN' are produced. # We want to get rid of these here: if buf[n-1] in {'n', 'N'}: - result = "nan" + result.add "nan" elif buf[n-1] == 'F': if buf[0] == '-': - result = "-inf" + result.add "-inf" else: - result = "inf" + result.add "inf" else: - result = $buf + result.add buf + +proc nimFloatToStr(f: float): string {.compilerproc.} = + result = newStringOfCap(8) + result.add f proc c_strtod(buf: cstring, endptr: ptr cstring): float64 {. importc: "strtod", header: "<stdlib.h>", noSideEffect.} @@ -469,22 +478,8 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat, number = c_strtod(t, nil) proc nimInt64ToStr(x: int64): string {.compilerRtl.} = - result = newString(sizeof(x)*4) - var i = 0 - var y = x - while true: - var d = y div 10 - result[i] = chr(abs(int(y - d*10)) + ord('0')) - inc(i) - y = d - if y == 0: break - if x < 0: - result[i] = '-' - inc(i) - setLen(result, i) - # mirror the string: - for j in 0..i div 2 - 1: - swap(result[j], result[i-j-1]) + result = newStringOfCap(sizeof(x)*4) + result.add x proc nimBoolToStr(x: bool): string {.compilerRtl.} = return if x: "true" else: "false" diff --git a/lib/upcoming/asyncdispatch.nim b/lib/upcoming/asyncdispatch.nim index 29b955c46..68ecbe81e 100644 --- a/lib/upcoming/asyncdispatch.nim +++ b/lib/upcoming/asyncdispatch.nim @@ -9,9 +9,9 @@ include "system/inclrtl" -import os, oids, tables, strutils, times, heapqueue +import os, oids, tables, strutils, times, heapqueue, lists -import nativesockets, net, queues +import nativesockets, net, deques export Port, SocketFlag @@ -135,7 +135,7 @@ include "../includes/asyncfutures" type PDispatcherBase = ref object of RootRef timers: HeapQueue[tuple[finishAt: float, fut: Future[void]]] - callbacks: Queue[proc ()] + callbacks: Deque[proc ()] proc processTimers(p: PDispatcherBase) {.inline.} = while p.timers.len > 0 and epochTime() >= p.timers[0].finishAt: @@ -143,7 +143,7 @@ proc processTimers(p: PDispatcherBase) {.inline.} = proc processPendingCallbacks(p: PDispatcherBase) = while p.callbacks.len > 0: - var cb = p.callbacks.dequeue() + var cb = p.callbacks.popFirst() cb() proc adjustedTimeout(p: PDispatcherBase, timeout: int): int {.inline.} = @@ -729,7 +729,7 @@ when defined(windows) or defined(nimdoc): var lpOutputBuf = newString(lpOutputLen) var dwBytesReceived: Dword let dwReceiveDataLength = 0.Dword # We don't want any data to be read. - let dwLocalAddressLength = Dword(sizeof (Sockaddr_in) + 16) + let dwLocalAddressLength = Dword(sizeof(Sockaddr_in) + 16) let dwRemoteAddressLength = Dword(sizeof(Sockaddr_in) + 16) template completeAccept() {.dirty.} = @@ -1095,9 +1095,11 @@ else: AsyncFD* = distinct cint Callback = proc (fd: AsyncFD): bool {.closure,gcsafe.} + DoublyLinkedListRef = ref DoublyLinkedList[Callback] + AsyncData = object - readCB: Callback - writeCB: Callback + readCBs: DoublyLinkedListRef + writeCBs: DoublyLinkedListRef AsyncEvent* = distinct SelectEvent @@ -1112,7 +1114,7 @@ else: new result result.selector = newSelector[AsyncData]() result.timers.newHeapQueue() - result.callbacks = initQueue[proc ()](64) + result.callbacks = initDeque[proc ()](64) var gDisp{.threadvar.}: PDispatcher ## Global dispatcher proc getGlobalDispatcher*(): PDispatcher = @@ -1121,7 +1123,10 @@ else: proc register*(fd: AsyncFD) = let p = getGlobalDispatcher() - var data = AsyncData() + var data = AsyncData( + readCBs: DoublyLinkedListRef(), + writeCBs: DoublyLinkedListRef() + ) p.selector.registerHandle(fd.SocketHandle, {}, data) proc newAsyncNativeSocket*(domain: cint, sockType: cint, @@ -1156,8 +1161,9 @@ else: let p = getGlobalDispatcher() var newEvents = {Event.Read} withData(p.selector, fd.SocketHandle, adata) do: - adata.readCB = cb - if adata.writeCB != nil: + adata.readCBs[].append(cb) + newEvents.incl(Event.Read) + if not isNil(adata.writeCBs.head): newEvents.incl(Event.Write) do: raise newException(ValueError, "File descriptor not registered.") @@ -1167,8 +1173,9 @@ else: let p = getGlobalDispatcher() var newEvents = {Event.Write} withData(p.selector, fd.SocketHandle, adata) do: - adata.writeCB = cb - if adata.readCB != nil: + adata.writeCBs[].append(cb) + newEvents.incl(Event.Write) + if not isNil(adata.readCBs.head): newEvents.incl(Event.Read) do: raise newException(ValueError, "File descriptor not registered.") @@ -1194,32 +1201,33 @@ else: var fd = keys[i].fd.SocketHandle let events = keys[i].events - if Event.Read in events: - let cb = keys[i].data.readCB - doAssert(cb != nil) - if cb(fd.AsyncFD): - p.selector.withData(fd, adata) do: - if adata.readCB == cb: - adata.readCB = nil - - if Event.Write in events: - let cb = keys[i].data.writeCB - doAssert(cb != nil) - if cb(fd.AsyncFD): - p.selector.withData(fd, adata) do: - if adata.writeCB == cb: - adata.writeCB = nil + if Event.Read in events or events == {Event.Error}: + for node in keys[i].data.readCBs[].nodes(): + let cb = node.value + if cb != nil: + if cb(fd.AsyncFD): + keys[i].data.readCBs[].remove(node) + else: + break + + if Event.Write in events or events == {Event.Error}: + for node in keys[i].data.writeCBs[].nodes(): + let cb = node.value + if cb != nil: + if cb(fd.AsyncFD): + keys[i].data.writeCBs[].remove(node) + else: + break when supportedPlatform: if (customSet * events) != {}: - let cb = keys[i].data.readCB - doAssert(cb != nil) - custom = true - if cb(fd.AsyncFD): - p.selector.withData(fd, adata) do: - if adata.readCB == cb: - adata.readCB = nil - p.selector.unregister(fd) + for node in keys[i].data.readCBs[].nodes(): + let cb = node.value + doAssert(cb != nil) + custom = true + if cb(fd.AsyncFD): + keys[i].data.readCBs[].remove(node) + p.selector.unregister(fd) # because state `data` can be modified in callback we need to update # descriptor events with currently registered callbacks. @@ -1227,8 +1235,8 @@ else: var update = false var newEvents: set[Event] = {} p.selector.withData(fd, adata) do: - if adata.readCB != nil: incl(newEvents, Event.Read) - if adata.writeCB != nil: incl(newEvents, Event.Write) + if not isNil(adata.readCBs.head): incl(newEvents, Event.Read) + if not isNil(adata.writeCBs.head): incl(newEvents, Event.Write) update = true if update: p.selector.updateHandle(fd, newEvents) @@ -1491,21 +1499,33 @@ else: ## ``oneshot`` - if ``true`` only one event will be dispatched, ## if ``false`` continuous events every ``timeout`` milliseconds. let p = getGlobalDispatcher() - var data = AsyncData(readCB: cb) + var data = AsyncData( + readCBs: DoublyLinkedListRef(), + writeCBs: DoublyLinkedListRef() + ) + data.readCBs[].append(cb) p.selector.registerTimer(timeout, oneshot, data) proc addSignal*(signal: int, cb: Callback) = ## Start watching signal ``signal``, and when signal appears, call the ## callback ``cb``. let p = getGlobalDispatcher() - var data = AsyncData(readCB: cb) + var data = AsyncData( + readCBs: DoublyLinkedListRef(), + writeCBs: DoublyLinkedListRef() + ) + data.readCBs[].append(cb) p.selector.registerSignal(signal, data) proc addProcess*(pid: int, cb: Callback) = ## Start watching for process exit with pid ``pid``, and then call ## the callback ``cb``. let p = getGlobalDispatcher() - var data = AsyncData(readCB: cb) + var data = AsyncData( + readCBs: DoublyLinkedListRef(), + writeCBs: DoublyLinkedListRef() + ) + data.readCBs[].append(cb) p.selector.registerProcess(pid, data) proc newAsyncEvent*(): AsyncEvent = @@ -1524,7 +1544,11 @@ else: ## Start watching for event ``ev``, and call callback ``cb``, when ## ev will be set to signaled state. let p = getGlobalDispatcher() - var data = AsyncData(readCB: cb) + var data = AsyncData( + readCBs: DoublyLinkedListRef(), + writeCBs: DoublyLinkedListRef() + ) + data.readCBs[].append(cb) p.selector.registerEvent(SelectEvent(ev), data) proc sleepAsync*(ms: int): Future[void] = @@ -1591,7 +1615,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = ## **Note**: This procedure is mostly used for testing. You likely want to ## use ``asyncnet.recvLine`` instead. - template addNLIfEmpty(): stmt = + template addNLIfEmpty(): typed = if result.len == 0: result.add("\c\L") @@ -1614,7 +1638,7 @@ proc recvLine*(socket: AsyncFD): Future[string] {.async.} = proc callSoon*(cbproc: proc ()) = ## Schedule `cbproc` to be called as soon as possible. ## The callback is called when control returns to the event loop. - getGlobalDispatcher().callbacks.enqueue(cbproc) + getGlobalDispatcher().callbacks.addLast(cbproc) proc runForever*() = ## Begins a never ending global dispatcher poll loop. diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 9dad7e489..241ad17ae 100644 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -26,7 +26,7 @@ when useWinVersion: from winlean import SocketHandle else: const - versions = "(|.10|.1.0.1|.1.0.0|.0.9.9|.0.9.8)" + versions = "(|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8)" when defined(macosx): const DLLSSLName = "libssl" & versions & ".dylib" @@ -261,11 +261,14 @@ proc ERR_error_string*(e: cInt, buf: cstring): cstring{.cdecl, proc ERR_get_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} proc ERR_peek_last_error*(): cInt{.cdecl, dynlib: DLLUtilName, importc.} -proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".} +when defined(android): + template OpenSSL_add_all_algorithms*() = discard +else: + proc OpenSSL_add_all_algorithms*(){.cdecl, dynlib: DLLUtilName, importc: "OPENSSL_add_all_algorithms_conf".} proc OPENSSL_config*(configName: cstring){.cdecl, dynlib: DLLSSLName, importc.} -when not useWinVersion and not defined(macosx): +when not useWinVersion and not defined(macosx) and not defined(android): proc CRYPTO_set_mem_functions(a,b,c: pointer){.cdecl, dynlib: DLLUtilName, importc.} @@ -279,7 +282,7 @@ when not useWinVersion and not defined(macosx): if p != nil: dealloc(p) proc CRYPTO_malloc_init*() = - when not useWinVersion and not defined(macosx): + when not useWinVersion and not defined(macosx) and not defined(android): CRYPTO_set_mem_functions(allocWrapper, reallocWrapper, deallocWrapper) proc SSL_CTX_ctrl*(ctx: SslCtx, cmd: cInt, larg: int, parg: pointer): int{. diff --git a/lib/wrappers/sqlite3.nim b/lib/wrappers/sqlite3.nim index e7fd2bc36..d2b70df8d 100644 --- a/lib/wrappers/sqlite3.nim +++ b/lib/wrappers/sqlite3.nim @@ -96,10 +96,6 @@ const #define SQLITE_TRANSIENT ((void(*)(void *))-1) SQLITE_DETERMINISTIC* = 0x800 -const - SQLITE_STATIC* = nil - SQLITE_TRANSIENT* = cast[pointer](- 1) - type Sqlite3 {.pure, final.} = object PSqlite3* = ptr Sqlite3 @@ -114,7 +110,7 @@ type Callback* = proc (para1: pointer, para2: int32, para3, para4: cstringArray): int32{.cdecl.} - Tbind_destructor_func* = proc (para1: pointer){.cdecl.} + Tbind_destructor_func* = proc (para1: pointer){.cdecl, locks: 0, tags: [].} Create_function_step_func* = proc (para1: Pcontext, para2: int32, para3: PValueArg){.cdecl.} Create_function_func_func* = proc (para1: Pcontext, para2: int32, @@ -132,6 +128,10 @@ type Tresult_func: Result_func, Tcreate_collation_func: Create_collation_func, Tcollation_needed_func: Collation_needed_func].} +const + SQLITE_STATIC* = nil + SQLITE_TRANSIENT* = cast[Tbind_destructor_func](-1) + proc close*(para1: PSqlite3): int32{.cdecl, dynlib: Lib, importc: "sqlite3_close".} proc exec*(para1: PSqlite3, sql: cstring, para3: Callback, para4: pointer, errmsg: var cstring): int32{.cdecl, dynlib: Lib, @@ -234,7 +234,8 @@ proc bind_parameter_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib, importc: "sqlite3_bind_parameter_name".} proc bind_parameter_index*(para1: Pstmt, zName: cstring): int32{.cdecl, dynlib: Lib, importc: "sqlite3_bind_parameter_index".} - #function sqlite3_clear_bindings(_para1:Psqlite3_stmt):longint;cdecl; external Sqlite3Lib name 'sqlite3_clear_bindings'; +proc clear_bindings*(para1: Pstmt): int32 {.cdecl, + dynlib: Lib, importc: "sqlite3_clear_bindings".} proc column_count*(pStmt: Pstmt): int32{.cdecl, dynlib: Lib, importc: "sqlite3_column_count".} proc column_name*(para1: Pstmt, para2: int32): cstring{.cdecl, dynlib: Lib, diff --git a/readme.md b/readme.md index 36d1ab0f9..b4a484f60 100644 --- a/readme.md +++ b/readme.md @@ -27,7 +27,8 @@ To build from source you will need: are: clang, Visual C++, Intel's C++ compiler * git or wget -**Note:** When installing ``gcc`` on Ubuntu (and likely other distros) ensure that the ``build-essentials`` package is installed also. +**Note:** When installing ``gcc`` on Ubuntu (and likely other distros) ensure +that the ``build-essentials`` package is installed also. If you are on a fairly modern *nix system, the following steps should work: @@ -58,7 +59,7 @@ source based installations where you added Nim's ``bin`` directory to your PATH the easiest way of installing Nimble is via: ``` -$ nim e install_nimble.nims +$ koch nimble ``` ## Community diff --git a/tests/assert/tfailedassert.nim b/tests/assert/tfailedassert.nim index 1e6764471..f0ca149f8 100644 --- a/tests/assert/tfailedassert.nim +++ b/tests/assert/tfailedassert.nim @@ -3,7 +3,7 @@ discard """ WARNING: false first assertion from bar ERROR: false second assertion from bar -1 -tests/assert/tfailedassert.nim:27 false assertion from foo +tfailedassert.nim:27 false assertion from foo ''' """ diff --git a/tests/async/tasyncall.nim b/tests/async/tasyncall.nim index 60ba557cc..63b2945a6 100644 --- a/tests/async/tasyncall.nim +++ b/tests/async/tasyncall.nim @@ -66,3 +66,12 @@ block: doAssert execTime * 100 < taskCount * sleepDuration doAssert results == expected + +block: + let + noIntFuturesFut = all(newSeq[Future[int]]()) + noVoidFuturesFut = all(newSeq[Future[void]]()) + + doAssert noIntFuturesFut.finished and not noIntFuturesFut.failed + doAssert noVoidFuturesFut.finished and not noVoidFuturesFut.failed + doAssert noIntFuturesFut.read() == @[] diff --git a/tests/async/tasynceverror.nim b/tests/async/tasynceverror.nim deleted file mode 100644 index dd05c831b..000000000 --- a/tests/async/tasynceverror.nim +++ /dev/null @@ -1,66 +0,0 @@ -discard """ - file: "tasynceverror.nim" - exitcode: 1 - outputsub: "Error: unhandled exception: " -""" -# error message is actually different on OSX -import - asyncdispatch, - asyncnet, - nativesockets, - os - - -const - testHost = "127.0.0.1" - testPort = Port(17357) - - -when defined(windows) or defined(nimdoc): - # TODO: just make it work on Windows for now. - quit("Error: unhandled exception: Connection reset by peer") -else: - proc createListenSocket(host: string, port: Port): TAsyncFD = - result = newAsyncNativeSocket() - - SocketHandle(result).setSockOptInt(SOL_SOCKET, SO_REUSEADDR, 1) - - var aiList = getAddrInfo(host, port, AF_INET) - if SocketHandle(result).bindAddr(aiList.ai_addr, aiList.ai_addrlen.Socklen) < 0'i32: - dealloc(aiList) - raiseOSError(osLastError()) - dealloc(aiList) - - if SocketHandle(result).listen(1) < 0'i32: - raiseOSError(osLastError()) - - - proc testAsyncSend() {.async.} = - var - ls = createListenSocket(testHost, testPort) - s = newAsyncSocket() - - await s.connect(testHost, testPort) - - var ps = await ls.accept() - closeSocket(ls) - - await ps.send("test 1", flags={}) - s.close() - # This send should raise EPIPE - await ps.send("test 2", flags={}) - SocketHandle(ps).close() - - - # The bug was, when the poll function handled EvError for us, - # our callbacks may never get executed, thus making the event - # loop block indefinitely. This is a timer to keep everything - # rolling. 400 ms is an arbitrary value, should be enough though. - proc timer() {.async.} = - await sleepAsync(400) - echo("Timer expired.") - quit(2) - - - asyncCheck(testAsyncSend()) - waitFor(timer()) diff --git a/tests/async/tgenericasync.nim b/tests/async/tgenericasync.nim new file mode 100644 index 000000000..ab704238a --- /dev/null +++ b/tests/async/tgenericasync.nim @@ -0,0 +1,14 @@ +discard """ + output: '''123 +abc''' +""" + +# bug #4856 + +import asyncdispatch + +proc say[T](t: T): Future[void] {.async.} = + echo $t + +waitFor(say(123)) +waitFor(say("abc")) diff --git a/tests/clearmsg/tmacroerrorproc.nim b/tests/clearmsg/tmacroerrorproc.nim new file mode 100644 index 000000000..9a6ff6a06 --- /dev/null +++ b/tests/clearmsg/tmacroerrorproc.nim @@ -0,0 +1,13 @@ +discard """ + file: "tmacroerrorproc.nim" + line: 13 + errormsg: "Expected a node of kind nnkCharLit, got nnkCommand" +""" +# issue #4915 +import macros + +macro mixer(n: typed): untyped = + expectKind(n, nnkCharLit) + +mixer: + echo "owh" \ No newline at end of file diff --git a/tests/collections/ttables.nim b/tests/collections/ttables.nim index 59fef4920..ef5ed92f5 100644 --- a/tests/collections/ttables.nim +++ b/tests/collections/ttables.nim @@ -95,9 +95,24 @@ block orderedTableTest1: for key, val in mpairs(t): val = 99 for val in mvalues(t): assert val == 99 +block orderedTableTest2: + var + s = initOrderedTable[string, int]() + t = initOrderedTable[string, int]() + assert s == t + for key, val in items(data): t[key] = val + assert s != t + for key, val in items(sorteddata): s[key] = val + assert s != t + t.clear() + assert s != t + for key, val in items(sorteddata): t[key] = val + assert s == t + block countTableTest1: var s = data.toTable var t = initCountTable[string]() + for k in s.keys: t.inc(k) for k in t.keys: assert t[k] == 1 t.inc("90", 3) @@ -115,6 +130,24 @@ block countTableTest1: else: break inc i +block countTableTest2: + var + s = initCountTable[int]() + t = initCountTable[int]() + assert s == t + s.inc(1) + assert s != t + t.inc(2) + assert s != t + t.inc(1) + assert s != t + s.inc(2) + assert s == t + s.inc(1) + assert s != t + t.inc(1) + assert s == t + block mpairsTableTest1: var t = initTable[string, int]() t["a"] = 1 @@ -134,6 +167,29 @@ block mpairsTableTest1: block SyntaxTest: var x = toTable[int, string]({:}) +block zeroHashKeysTest: + proc doZeroHashValueTest[T, K, V](t: T, nullHashKey: K, value: V) = + let initialLen = t.len + var testTable = t + testTable[nullHashKey] = value + assert testTable[nullHashKey] == value + assert testTable.len == initialLen + 1 + testTable.del(nullHashKey) + assert testTable.len == initialLen + + # with empty table + doZeroHashValueTest(toTable[int,int]({:}), 0, 42) + doZeroHashValueTest(toTable[string,int]({:}), "", 23) + doZeroHashValueTest(toOrderedTable[int,int]({:}), 0, 42) + doZeroHashValueTest(toOrderedTable[string,int]({:}), "", 23) + + # with non-empty table + doZeroHashValueTest(toTable[int,int]({1:2}), 0, 42) + doZeroHashValueTest(toTable[string,string]({"foo": "bar"}), "", "zero") + doZeroHashValueTest(toOrderedTable[int,int]({3:4}), 0, 42) + doZeroHashValueTest(toOrderedTable[string,string]({"egg": "sausage"}), + "", "spam") + # Until #4448 is fixed, these tests will fail when false: block clearTableTest: diff --git a/tests/generics/tstatictalias.nim b/tests/generics/tstatictalias.nim new file mode 100644 index 000000000..98751b8cb --- /dev/null +++ b/tests/generics/tstatictalias.nim @@ -0,0 +1,20 @@ +discard """ + output: '''G:0,1:0.1 +G:0,1:0.1 +H:1:0.1''' +""" + +type + G[i,j:static[int]] = object + v:float + H[j:static[int]] = G[0,j] +proc p[i,j:static[int]](x:G[i,j]) = echo "G:",i,",",j,":",x.v +proc q[j:static[int]](x:H[j]) = echo "H:",j,":",x.v + +var + g0 = G[0,1](v: 0.1) + h0:H[1] = g0 +p(g0) +p(h0) +q(h0) +# bug #4863 diff --git a/tests/js/tconsole.nim b/tests/js/tconsole.nim new file mode 100644 index 000000000..f6da71c20 --- /dev/null +++ b/tests/js/tconsole.nim @@ -0,0 +1,13 @@ +discard """ + output: '''Hello, console +1 2 3 +1 'hi' 1.1''' +""" + +# This file tests the JavaScript console + +import jsconsole + +console.log("Hello, console") +console.log(1, 2, 3) +console.log(1, "hi", 1.1) \ No newline at end of file diff --git a/tests/macros/tdump.nim b/tests/macros/tdump.nim new file mode 100644 index 000000000..e4c14dc6b --- /dev/null +++ b/tests/macros/tdump.nim @@ -0,0 +1,13 @@ +discard """ + output: '''x = 10 +x + y = 30 +''' +""" + +import future + +let + x = 10 + y = 20 +dump x +dump(x + y) \ No newline at end of file diff --git a/tests/manyloc/keineschweine/keineschweine.nim b/tests/manyloc/keineschweine/keineschweine.nim index 49c0a2476..804a22852 100644 --- a/tests/manyloc/keineschweine/keineschweine.nim +++ b/tests/manyloc/keineschweine/keineschweine.nim @@ -40,7 +40,7 @@ type trailDelay*: float body: chipmunk.PBody shape: chipmunk.PShape -import vehicles +include vehicles const LGrabbable* = (1 shl 0).TLayers LBorders* = (1 shl 1).TLayers diff --git a/tests/manyloc/keineschweine/lib/vehicles.nim b/tests/manyloc/keineschweine/lib/vehicles.nim index ddfb43b38..e245c9e8c 100644 --- a/tests/manyloc/keineschweine/lib/vehicles.nim +++ b/tests/manyloc/keineschweine/lib/vehicles.nim @@ -1,6 +1,6 @@ import sfml, chipmunk, - sg_assets, sfml_stuff, "../keineschweine" + sg_assets, sfml_stuff#, "../keineschweine" proc accel*(obj: PVehicle, dt: float) = diff --git a/tests/method/tmapper.nim b/tests/method/tmapper.nim index 75b36e69a..0008d9033 100644 --- a/tests/method/tmapper.nim +++ b/tests/method/tmapper.nim @@ -1,5 +1,5 @@ discard """ - errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tests/method/tmapper.nim(22,7)" + errormsg: "invalid declaration order; cannot attach 'step' to method defined here: tmapper.nim(22,7)" line: 25 """ diff --git a/tests/misc/tvarious1.nim b/tests/misc/tvarious1.nim index 1d5ad876a..595c77919 100644 --- a/tests/misc/tvarious1.nim +++ b/tests/misc/tvarious1.nim @@ -18,15 +18,15 @@ echo v[2] # bug #569 -import queues +import deques type TWidget = object - names: Queue[string] + names: Deque[string] -var w = TWidget(names: initQueue[string]()) +var w = TWidget(names: initDeque[string]()) -add(w.names, "Whopie") +addLast(w.names, "Whopie") for n in w.names: echo(n) diff --git a/tests/modules/trecinca.nim b/tests/modules/trecinca.nim index 14a91ba5c..7a74d7a46 100644 --- a/tests/modules/trecinca.nim +++ b/tests/modules/trecinca.nim @@ -1,7 +1,7 @@ discard """ - file: "tests/reject/trecincb.nim" + file: "trecincb.nim" line: 9 - errormsg: "recursive dependency: 'tests/modules/trecincb.nim'" + errormsg: "recursive dependency: 'trecincb.nim'" """ # Test recursive includes diff --git a/tests/modules/trecincb.nim b/tests/modules/trecincb.nim index 299a242e1..1d3eb5503 100644 --- a/tests/modules/trecincb.nim +++ b/tests/modules/trecincb.nim @@ -1,7 +1,7 @@ discard """ file: "trecincb.nim" line: 9 - errormsg: "recursive dependency: 'tests/modules/trecincb.nim'" + errormsg: "recursive dependency: 'trecincb.nim'" """ # Test recursive includes diff --git a/tests/modules/trecmod.nim b/tests/modules/trecmod.nim index d567e293b..5f053bcae 100644 --- a/tests/modules/trecmod.nim +++ b/tests/modules/trecmod.nim @@ -1,2 +1,8 @@ +discard """ + file: "mrecmod.nim" + line: 1 + errormsg: "recursive module dependency detected" + disabled: true +""" # recursive module import mrecmod diff --git a/tests/modules/trecmod2.nim b/tests/modules/trecmod2.nim index 85fe2215f..03c8cf70d 100644 --- a/tests/modules/trecmod2.nim +++ b/tests/modules/trecmod2.nim @@ -1,10 +1,13 @@ +discard """ + output: "4" +""" type T1* = int # Module A exports the type ``T1`` import mrecmod2 # the compiler starts parsing B - +# the manual says this should work proc main() = - var i = p(3) # works because B has been parsed completely here + echo p(3) # works because B has been parsed completely here main() diff --git a/tests/osproc/tworkingdir.nim b/tests/osproc/tworkingdir.nim new file mode 100644 index 000000000..84ba3375c --- /dev/null +++ b/tests/osproc/tworkingdir.nim @@ -0,0 +1,16 @@ +discard """ + file: "tworkingdir.nim" + output: "" +""" + +import osproc, os +when defined(windows): + # Windows don't have this issue, so we won't test it. + discard +else: + let dir1 = getCurrentDir() + var process = startProcess("/usr/bin/env", "/usr/bin", ["true"]) + let dir2 = getCurrentDir() + discard process.waitForExit() + process.close() + doAssert(dir1 == dir2, $dir1 & " != " & $dir2) diff --git a/tests/parser/tbraces.nim b/tests/parser/tbraces.nim new file mode 100644 index 000000000..86c854546 --- /dev/null +++ b/tests/parser/tbraces.nim @@ -0,0 +1,432 @@ +#? braces + +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This module implements some common generic algorithms. + +type + SortOrder* = enum { ## sort order + Descending, Ascending + } + + +type( + DummyAlias = int + OtherAlias = distinct char + + SomeObject = object of RootObj { ## declaration here + fieldA, fieldB: int + case order: SortOrder { + of Descending {a: string} + of Ascending {b: seq[char]} + } + } + + MyConcept = concept x { + x is int + } +) + +{.deprecated: [TSortOrder: SortOrder].} + + +proc `*`*(x: int, order: SortOrder): int @inline { + ## flips `x` if ``order == Descending``; + ## if ``order == Ascending`` then `x` is returned. + ## `x` is supposed to be the result of a comparator, ie ``< 0`` for + ## *less than*, ``== 0`` for *equal*, ``> 0`` for *greater than*. + var y = order.ord - 1 + result = (x xor y) - y +} + +proc fill*[T](a: var openArray[T], first, last: Natural, value: T) { + ## fills the array ``a[first..last]`` with `value`. + var x = first + while x <= last { + a[x] = value + inc(x) + } +} + +proc fill*[T](a: var openArray[T], value: T) { + ## fills the array `a` with `value`. + fill(a, 0, a.high, value) +} + +proc reverse*[T](a: var openArray[T], first, last: Natural) { + ## reverses the array ``a[first..last]``. + var x = first + var y = last + while x < y { + swap(a[x], a[y]) + dec(y) + inc(x) + } +} + +proc reverse*[T](a: var openArray[T]) { + ## reverses the array `a`. + reverse(a, 0, a.high) +} + +proc reversed*[T](a: openArray[T], first: Natural, last: int): seq[T] { + ## returns the reverse of the array `a[first..last]`. + assert last >= first-1 + var i = last - first + var x = first.int + result = newSeq[T](i + 1) + while i >= 0 { + result[i] = a[x] + dec(i) + inc(x) + } +} + +proc reversed*[T](a: openArray[T]): seq[T] { + ## returns the reverse of the array `a`. + reversed(a, 0, a.high) +} + +proc binarySearch*[T](a: openArray[T], key: T): int { + ## binary search for `key` in `a`. Returns -1 if not found. + var b = len(a) + while result < b { + var mid = (result + b) div 2 + if a[mid] < key { result = mid + 1 } else { b = mid } + } + if result >= len(a) or a[result] != key { result = -1 } +} + +proc smartBinarySearch*[T](a: openArray[T], key: T): int { + ## ``a.len`` must be a power of 2 for this to work. + var step = a.len div 2 + while step > 0 { + if a[result or step] <= key { result = result or step } + step = step shr 1 + } + if a[result] != key { result = -1 } +} + +const ( + onlySafeCode = true +) + +proc lowerBound*[T](a: openArray[T], key: T, cmp: proc(x,y: T): int @closure): int { + ## same as binarySearch except that if key is not in `a` then this + ## returns the location where `key` would be if it were. In other + ## words if you have a sorted sequence and you call + ## insert(thing, elm, lowerBound(thing, elm)) + ## the sequence will still be sorted. + ## + ## `cmp` is the comparator function to use, the expected return values are + ## the same as that of system.cmp. + ## + ## example:: + ## + ## var arr = @[1,2,3,5,6,7,8,9] + ## arr.insert(4, arr.lowerBound(4)) + ## # after running the above arr is `[1,2,3,4,5,6,7,8,9]` + result = a.low + var count = a.high - a.low + 1 + var step, pos: int + while count != 0 { + step = count div 2 + pos = result + step + if cmp(a[pos], key) < 0 { + result = pos + 1 + count -= step + 1 + } else { + count = step + } + } +} + +proc lowerBound*[T](a: openArray[T], key: T): int { lowerBound(a, key, cmp[T]) } +proc merge[T](a, b: var openArray[T], lo, m, hi: int, + cmp: proc (x, y: T): int @closure, order: SortOrder) { + template `<-` (a, b) { + when false { + a = b + } elif onlySafeCode { + shallowCopy(a, b) + } else { + copyMem(addr(a), addr(b), sizeof(T)) + } + } + # optimization: If max(left) <= min(right) there is nothing to do! + # 1 2 3 4 ## 5 6 7 8 + # -> O(n) for sorted arrays. + # On random data this safes up to 40% of merge calls + if cmp(a[m], a[m+1]) * order <= 0 { return } + var j = lo + # copy a[j..m] into b: + assert j <= m + when onlySafeCode { + var bb = 0 + while j <= m { + b[bb] <- a[j] + inc(bb) + inc(j) + } + } else { + copyMem(addr(b[0]), addr(a[j]), sizeof(T)*(m-j+1)) + j = m+1 + } + var i = 0 + var k = lo + # copy proper element back: + while k < j and j <= hi { + if cmp(b[i], a[j]) * order <= 0 { + a[k] <- b[i] + inc(i) + } else { + a[k] <- a[j] + inc(j) + } + inc(k) + } + # copy rest of b: + when onlySafeCode { + while k < j { + a[k] <- b[i] + inc(k) + inc(i) + } + } else { + if k < j { copyMem(addr(a[k]), addr(b[i]), sizeof(T)*(j-k)) } + } +} + +proc sort*[T](a: var openArray[T], + cmp: proc (x, y: T): int @closure, + order = SortOrder.Ascending) { + ## Default Nim sort (an implementation of merge sort). The sorting + ## is guaranteed to be stable and the worst case is guaranteed to + ## be O(n log n). + ## The current implementation uses an iterative + ## mergesort to achieve this. It uses a temporary sequence of + ## length ``a.len div 2``. Currently Nim does not support a + ## sensible default argument for ``cmp``, so you have to provide one + ## of your own. However, the ``system.cmp`` procs can be used: + ## + ## .. code-block:: nim + ## + ## sort(myIntArray, system.cmp[int]) + ## + ## # do not use cmp[string] here as we want to use the specialized + ## # overload: + ## sort(myStrArray, system.cmp) + ## + ## You can inline adhoc comparison procs with the `do notation + ## <manual.html#procedures-do-notation>`_. Example: + ## + ## .. code-block:: nim + ## + ## people.sort do (x, y: Person) -> int: + ## result = cmp(x.surname, y.surname) + ## if result == 0: + ## result = cmp(x.name, y.name) + var n = a.len + var b: seq[T] + newSeq(b, n div 2) + var s = 1 + while s < n { + var m = n-1-s + while m >= 0 { + merge(a, b, max(m-s+1, 0), m, m+s, cmp, order) + dec(m, s*2) + } + s = s*2 + } +} + +proc sorted*[T](a: openArray[T], cmp: proc(x, y: T): int {.closure.}, + order = SortOrder.Ascending): seq[T] { + ## returns `a` sorted by `cmp` in the specified `order`. + result = newSeq[T](a.len) + for i in 0 .. a.high { result[i] = a[i] } + sort(result, cmp, order) +} + +template sortedByIt*(seq1, op: untyped): untyped { + ## Convenience template around the ``sorted`` proc to reduce typing. + ## + ## The template injects the ``it`` variable which you can use directly in an + ## expression. Example: + ## + ## .. code-block:: nim + ## + ## type Person = tuple[name: string, age: int] + ## var + ## p1: Person = (name: "p1", age: 60) + ## p2: Person = (name: "p2", age: 20) + ## p3: Person = (name: "p3", age: 30) + ## p4: Person = (name: "p4", age: 30) + ## people = @[p1,p2,p4,p3] + ## + ## echo people.sortedByIt(it.name) + ## + ## Because the underlying ``cmp()`` is defined for tuples you can do + ## a nested sort like in the following example: + ## + ## .. code-block:: nim + ## + ## echo people.sortedByIt((it.age, it.name)) + ## + var result = sorted(seq1, proc(x, y: type[seq1[0]]): int { + var it {.inject.} = x + let a = op + it = y + let b = op + result = cmp(a, b) + }) + result +} + +proc isSorted*[T](a: openarray[T], + cmp: proc(x, y: T): int {.closure.}, + order = SortOrder.Ascending): bool { + ## Checks to see whether `a` is already sorted in `order` + ## using `cmp` for the comparison. Parameters identical + ## to `sort` + result = true + for i in 0..<len(a)-1 { + case cmp(a[i],a[i+1]) * order > 0 { + of true { return false } + of false {} + } + } +} + +proc product*[T](x: openArray[seq[T]]): seq[seq[T]] { + ## produces the Cartesian product of the array. Warning: complexity + ## may explode. + result = newSeq[seq[T]]() + if x.len == 0 { return } + if x.len == 1 { + result = @x + return + } + var ( + indexes = newSeq[int](x.len) + initial = newSeq[int](x.len) + index = 0 + ) + var next = newSeq[T]() + next.setLen(x.len) + for i in 0..(x.len-1) { + if len(x[i]) == 0 { return } + initial[i] = len(x[i])-1 + } + indexes = initial + while true { + while indexes[index] == -1 { + indexes[index] = initial[index] + index += 1 + if index == x.len { return } + indexes[index] -= 1 + } + for ni, i in indexes { + next[ni] = x[ni][i] + } + var res: seq[T] + shallowCopy(res, next) + result.add(res) + index = 0 + indexes[index] -= 1 + } +} + +proc nextPermutation*[T](x: var openarray[T]): bool {.discardable.} { + ## Calculates the next lexicographic permutation, directly modifying ``x``. + ## The result is whether a permutation happened, otherwise we have reached + ## the last-ordered permutation. + ## + ## .. code-block:: nim + ## + ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ## v.nextPermutation() + ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] + if x.len < 2 { + return false } + + var i = x.high + while i > 0 and x[i-1] >= x[i] { dec i } + if i == 0 { return false } + + var j = x.high + while j >= i and x[j] <= x[i-1] { dec j } + + swap x[j], x[i-1] + x.reverse(i, x.high) + + result = true +} + +proc prevPermutation*[T](x: var openarray[T]): bool @discardable { + ## Calculates the previous lexicographic permutation, directly modifying + ## ``x``. The result is whether a permutation happened, otherwise we have + ## reached the first-ordered permutation. + ## + ## .. code-block:: nim + ## + ## var v = @[0, 1, 2, 3, 4, 5, 6, 7, 9, 8] + ## v.prevPermutation() + ## echo v # @[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + if x.len < 2 { return false } + + var i = x.high + while i > 0 and x[i-1] <= x[i] { + dec i + } + if i == 0 { return false } + + x.reverse(i, x.high) + + var j = x.high + while j >= i and x[j-1] < x[i-1] { + dec j + } + swap x[i-1], x[j] + + result = true +} + +when isMainModule { + # Tests for lowerBound + var arr = @[1,2,3,5,6,7,8,9] + assert arr.lowerBound(0) == 0 + assert arr.lowerBound(4) == 3 + assert arr.lowerBound(5) == 3 + assert arr.lowerBound(10) == 8 + arr = @[1,5,10] + try { + assert arr.lowerBound(4) == 1 + assert arr.lowerBound(5) == 1 + assert arr.lowerBound(6) == 2 + } except ValueError {} + # Tests for isSorted + var srt1 = [1,2,3,4,4,4,4,5] + var srt2 = ["iello","hello"] + var srt3 = [1.0,1.0,1.0] + var srt4: seq[int] = @[] + assert srt1.isSorted(cmp) == true + assert srt2.isSorted(cmp) == false + assert srt3.isSorted(cmp) == true + var srtseq = newSeq[int]() + assert srtseq.isSorted(cmp) == true + # Tests for reversed + var arr1 = @[0,1,2,3,4] + assert arr1.reversed() == @[4,3,2,1,0] + for i in 0 .. high(arr1) { + assert arr1.reversed(0, i) == arr1.reversed()[high(arr1) - i .. high(arr1)] + assert arr1.reversed(i, high(arr1)) == arr1.reversed()[0 .. high(arr1) - i] + } +} diff --git a/tests/stdlib/tgetfileinfo.nim b/tests/stdlib/tgetfileinfo.nim index 8a0538a5f..1c897b702 100644 --- a/tests/stdlib/tgetfileinfo.nim +++ b/tests/stdlib/tgetfileinfo.nim @@ -1,5 +1,5 @@ discard """ - output: "" + output: "pcDir\npcFile\npcLinkToDir\npcLinkToFile\n" """ import os, strutils @@ -93,4 +93,42 @@ proc testGetFileInfo = discard #echo("Handle : Invalid File : Success") + # Test kind for files, directories and symlinks. + block: + let + tmp = getTempDir() + dirPath = tmp / "test-dir" + filePath = tmp / "test-file" + linkDirPath = tmp / "test-link-dir" + linkFilePath = tmp / "test-link-file" + + createDir(dirPath) + writeFile(filePath, "") + when defined(posix): + createSymlink(dirPath, linkDirPath) + createSymlink(filePath, linkFilePath) + + let + dirInfo = getFileInfo(dirPath) + fileInfo = getFileInfo(filePath) + when defined(posix): + let + linkDirInfo = getFileInfo(linkDirPath, followSymlink = false) + linkFileInfo = getFileInfo(linkFilePath, followSymlink = false) + + echo dirInfo.kind + echo fileInfo.kind + when defined(posix): + echo linkDirInfo.kind + echo linkFileInfo.kind + else: + echo pcLinkToDir + echo pcLinkToFile + + removeDir(dirPath) + removeFile(filePath) + when defined(posix): + removeFile(linkDirPath) + removeFile(linkFilePath) + testGetFileInfo() diff --git a/tests/stdlib/tmitems.nim b/tests/stdlib/tmitems.nim index c713d91a4..17265e1f7 100644 --- a/tests/stdlib/tmitems.nim +++ b/tests/stdlib/tmitems.nim @@ -98,13 +98,13 @@ block: x += 10 echo sl -import queues +import deques block: - var q = initQueue[int]() - q.add(1) - q.add(2) - q.add(3) + var q = initDeque[int]() + q.addLast(1) + q.addLast(2) + q.addLast(3) for x in q.mitems: x += 10 echo q diff --git a/tests/stdlib/tnilecho.nim b/tests/stdlib/tnilecho.nim new file mode 100644 index 000000000..147b5e492 --- /dev/null +++ b/tests/stdlib/tnilecho.nim @@ -0,0 +1,2 @@ +var x = @["1", nil, "3"] +doAssert $x == "@[1, nil, 3]" diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim index 1ddaacfcb..1ef02f97e 100644 --- a/tests/stdlib/tos.nim +++ b/tests/stdlib/tos.nim @@ -36,6 +36,9 @@ false false false false +true +true +Raises ''' """ # test os path creation, iteration, and deletion @@ -86,3 +89,22 @@ for file in files: removeDir(dname) echo dirExists(dname) + +# createDir should create recursive directories +createDir(dirs[0] / dirs[1]) +echo dirExists(dirs[0] / dirs[1]) # true +removeDir(dirs[0]) + +# createDir should properly handle trailing separator +createDir(dname / "") +echo dirExists(dname) # true +removeDir(dname) + +# createDir should raise IOError if the path exists +# and is not a directory +open(dname, fmWrite).close +try: + createDir(dname) +except IOError: + echo "Raises" +removeFile(dname) diff --git a/tests/stdlib/ttime.nim b/tests/stdlib/ttime.nim index 3ab287c4e..b28d8aecd 100644 --- a/tests/stdlib/ttime.nim +++ b/tests/stdlib/ttime.nim @@ -9,77 +9,97 @@ import # $ date --date='@2147483647' # Tue 19 Jan 03:14:07 GMT 2038 -var t = getGMTime(fromSeconds(2147483647)) -doAssert t.format("ddd dd MMM hh:mm:ss ZZZ yyyy") == "Tue 19 Jan 03:14:07 UTC 2038" -doAssert t.format("ddd ddMMMhh:mm:ssZZZyyyy") == "Tue 19Jan03:14:07UTC2038" - -doAssert t.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 0 00 00:00 UTC" - -doAssert t.format("yyyyMMddhhmmss") == "20380119031407" - -var t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 -doAssert t2.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 0 00 00:00 UTC" +proc checkFormat(t: TimeInfo, format, expected: string) = + let actual = t.format(format) + if actual != expected: + echo "Formatting failure!" + echo "expected: ", expected + echo "actual : ", actual + doAssert false + +let t = getGMTime(fromSeconds(2147483647)) +t.checkFormat("ddd dd MMM hh:mm:ss yyyy", "Tue 19 Jan 03:14:07 2038") +t.checkFormat("ddd ddMMMhh:mm:ssyyyy", "Tue 19Jan03:14:072038") +t.checkFormat("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz", + "19 19 Tue Tuesday 3 03 3 03 14 14 1 01 Jan January 7 07 A AM 8 38 038 2038 02038 +0 +00 +00:00") + +t.checkFormat("yyyyMMddhhmmss", "20380119031407") + +let t2 = getGMTime(fromSeconds(160070789)) # Mon 27 Jan 16:06:29 GMT 1975 +t2.checkFormat("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz", + "27 27 Mon Monday 4 04 16 16 6 06 1 01 Jan January 29 29 P PM 5 75 975 1975 01975 +0 +00 +00:00") when not defined(JS): when sizeof(Time) == 8: var t3 = getGMTime(fromSeconds(889067643645)) # Fri 7 Jun 19:20:45 BST 30143 - doAssert t3.format("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & - " ss t tt y yy yyy yyyy yyyyy z zz zzz ZZZ") == - "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 0 00 00:00 UTC" - doAssert t3.format(":,[]()-/") == ":,[]()-/" + t3.checkFormat("d dd ddd dddd h hh H HH m mm M MM MMM MMMM s" & + " ss t tt y yy yyy yyyy yyyyy z zz zzz", + "7 07 Fri Friday 6 06 18 18 20 20 6 06 Jun June 45 45 P PM 3 43 143 0143 30143 +0 +00 +00:00") + t3.checkFormat(":,[]()-/", ":,[]()-/") var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 -doAssert t4.format("M MM MMM MMMM") == "10 10 Oct October" +t4.checkFormat("M MM MMM MMMM", "10 10 Oct October") # Interval tests -doAssert((t4 - initInterval(years = 2)).format("yyyy") == "1995") -doAssert((t4 - initInterval(years = 7, minutes = 34, seconds = 24)).format("yyyy mm ss") == "1990 24 10") +(t4 - initInterval(years = 2)).checkFormat("yyyy", "1995") +(t4 - initInterval(years = 7, minutes = 34, seconds = 24)).checkFormat("yyyy mm ss", "1990 24 10") proc parseTest(s, f, sExpected: string, ydExpected: int) = - let parsed = s.parse(f) - doAssert($parsed == sExpected) + let + parsed = s.parse(f) + parsedStr = $getGMTime(toTime(parsed)) + if parsedStr != sExpected: + echo "Parsing failure!" + echo "expected: ", sExpected + echo "actual : ", parsedStr + doAssert false doAssert(parsed.yearday == ydExpected) proc parseTestTimeOnly(s, f, sExpected: string) = doAssert(sExpected in $s.parse(f)) -parseTest("Tuesday at 09:04am on Dec 15, 2015", - "dddd at hh:mmtt on MMM d, yyyy", "Tue Dec 15 09:04:00 2015", 348) +# because setting a specific timezone for testing is platform-specific, we use +# explicit timezone offsets in all tests. + +parseTest("Tuesday at 09:04am on Dec 15, 2015 +0", + "dddd at hh:mmtt on MMM d, yyyy z", "2015-12-15T09:04:00+00:00", 348) # ANSIC = "Mon Jan _2 15:04:05 2006" -parseTest("Thu Jan 12 15:04:05 2006", "ddd MMM dd HH:mm:ss yyyy", - "Thu Jan 12 15:04:05 2006", 11) +parseTest("Thu Jan 12 15:04:05 2006 +0", "ddd MMM dd HH:mm:ss yyyy z", + "2006-01-12T15:04:05+00:00", 11) # UnixDate = "Mon Jan _2 15:04:05 MST 2006" -parseTest("Thu Jan 12 15:04:05 MST 2006", "ddd MMM dd HH:mm:ss ZZZ yyyy", - "Thu Jan 12 15:04:05 2006", 11) +parseTest("Thu Jan 12 15:04:05 2006 +0", "ddd MMM dd HH:mm:ss yyyy z", + "2006-01-12T15:04:05+00:00", 11) # RubyDate = "Mon Jan 02 15:04:05 -0700 2006" -parseTest("Mon Feb 29 15:04:05 -07:00 2016", "ddd MMM dd HH:mm:ss zzz yyyy", - "Mon Feb 29 15:04:05 2016", 59) # leap day +parseTest("Mon Feb 29 15:04:05 -07:00 2016 +0", "ddd MMM dd HH:mm:ss zzz yyyy z", + "2016-02-29T15:04:05+00:00", 59) # leap day # RFC822 = "02 Jan 06 15:04 MST" -parseTest("12 Jan 16 15:04 MST", "dd MMM yy HH:mm ZZZ", - "Tue Jan 12 15:04:00 2016", 11) +parseTest("12 Jan 16 15:04 +0", "dd MMM yy HH:mm z", + "2016-01-12T15:04:00+00:00", 11) # RFC822Z = "02 Jan 06 15:04 -0700" # RFC822 with numeric zone parseTest("01 Mar 16 15:04 -07:00", "dd MMM yy HH:mm zzz", - "Tue Mar 1 15:04:00 2016", 60) # day after february in leap year + "2016-03-01T22:04:00+00:00", 60) # day after february in leap year # RFC850 = "Monday, 02-Jan-06 15:04:05 MST" -parseTest("Monday, 12-Jan-06 15:04:05 MST", "dddd, dd-MMM-yy HH:mm:ss ZZZ", - "Thu Jan 12 15:04:05 2006", 11) +parseTest("Monday, 12-Jan-06 15:04:05 +0", "dddd, dd-MMM-yy HH:mm:ss z", + "2006-01-12T15:04:05+00:00", 11) # RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" -parseTest("Sun, 01 Mar 2015 15:04:05 MST", "ddd, dd MMM yyyy HH:mm:ss ZZZ", - "Sun Mar 1 15:04:05 2015", 59) # day after february in non-leap year +parseTest("Sun, 01 Mar 2015 15:04:05 +0", "ddd, dd MMM yyyy HH:mm:ss z", + "2015-03-01T15:04:05+00:00", 59) # day after february in non-leap year # RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" # RFC1123 with numeric zone parseTest("Thu, 12 Jan 2006 15:04:05 -07:00", "ddd, dd MMM yyyy HH:mm:ss zzz", - "Thu Jan 12 15:04:05 2006", 11) + "2006-01-12T22:04:05+00:00", 11) # RFC3339 = "2006-01-02T15:04:05Z07:00" parseTest("2006-01-12T15:04:05Z-07:00", "yyyy-MM-ddTHH:mm:ssZzzz", - "Thu Jan 12 15:04:05 2006", 11) + "2006-01-12T22:04:05+00:00", 11) parseTest("2006-01-12T15:04:05Z-07:00", "yyyy-MM-dd'T'HH:mm:ss'Z'zzz", - "Thu Jan 12 15:04:05 2006", 11) + "2006-01-12T22:04:05+00:00", 11) # RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" parseTest("2006-01-12T15:04:05.999999999Z-07:00", - "yyyy-MM-ddTHH:mm:ss.999999999Zzzz", "Thu Jan 12 15:04:05 2006", 11) + "yyyy-MM-ddTHH:mm:ss.999999999Zzzz", "2006-01-12T22:04:05+00:00", 11) +for tzFormat in ["z", "zz", "zzz"]: + # formatting timezone as 'Z' for UTC + parseTest("2001-01-12T22:04:05Z", "yyyy-MM-dd'T'HH:mm:ss" & tzFormat, + "2001-01-12T22:04:05+00:00", 11) # Kitchen = "3:04PM" parseTestTimeOnly("3:04PM", "h:mmtt", "15:04:00") #when not defined(testing): @@ -101,21 +121,20 @@ doAssert getDayOfWeekJulian(21, 9, 1970) == dMon doAssert getDayOfWeekJulian(1, 1, 2000) == dSat doAssert getDayOfWeekJulian(1, 1, 2021) == dFri -# toSeconds tests with GM and Local timezones -#var t4 = getGMTime(fromSeconds(876124714)) # Mon 6 Oct 08:58:34 BST 1997 -var t4L = getLocalTime(fromSeconds(876124714)) -doAssert toSeconds(timeInfoToTime(t4L)) == 876124714 # fromSeconds is effectively "localTime" -doAssert toSeconds(timeInfoToTime(t4L)) + t4L.timezone.float == toSeconds(timeInfoToTime(t4)) +# toSeconds tests with GM timezone +let t4L = getGMTime(fromSeconds(876124714)) +doAssert toSeconds(toTime(t4L)) == 876124714 +doAssert toSeconds(toTime(t4L)) + t4L.timezone.float == toSeconds(toTime(t4)) # adding intervals var - a1L = toSeconds(timeInfoToTime(t4L + initInterval(hours = 1))) + t4L.timezone.float - a1G = toSeconds(timeInfoToTime(t4)) + 60.0 * 60.0 + a1L = toSeconds(toTime(t4L + initInterval(hours = 1))) + t4L.timezone.float + a1G = toSeconds(toTime(t4)) + 60.0 * 60.0 doAssert a1L == a1G # subtracting intervals -a1L = toSeconds(timeInfoToTime(t4L - initInterval(hours = 1))) + t4L.timezone.float -a1G = toSeconds(timeInfoToTime(t4)) - (60.0 * 60.0) +a1L = toSeconds(toTime(t4L - initInterval(hours = 1))) + t4L.timezone.float +a1G = toSeconds(toTime(t4)) - (60.0 * 60.0) doAssert a1L == a1G # add/subtract TimeIntervals and Time/TimeInfo @@ -140,4 +159,71 @@ doAssert initInterval(months = 13) == initInterval(months = 1, years = 1) # Bug with adding a day to a Time let day = 24.hours let tomorrow = getTime() + day -doAssert tomorrow - getTime() == 60*60*24 \ No newline at end of file +doAssert tomorrow - getTime() == 60*60*24 + +doAssert milliseconds(1000 * 60) == minutes(1) +doAssert milliseconds(1000 * 60 * 60) == hours(1) +doAssert milliseconds(1000 * 60 * 60 * 24) == days(1) +doAssert seconds(60 * 60) == hours(1) +doAssert seconds(60 * 60 * 24) == days(1) +doAssert seconds(60 * 60 + 65) == (hours(1) + minutes(1) + seconds(5)) + +# Bug with parse not setting DST properly if the current local DST differs from +# the date being parsed. Need to test parse dates both in and out of DST. We +# are testing that be relying on the fact that tranforming a TimeInfo to a Time +# and back again will correctly set the DST value. With the incorrect parse +# behavior this will introduce a one hour offset from the named time and the +# parsed time if the DST value differs between the current time and the date we +# are parsing. +# +# Unfortunately these tests depend on the locale of the system in which they +# are run. They will not be meaningful when run in a locale without DST. They +# also assume that Jan. 1 and Jun. 1 will have differing isDST values. +let dstT1 = parse("2016-01-01 00:00:00", "yyyy-MM-dd HH:mm:ss") +let dstT2 = parse("2016-06-01 00:00:00", "yyyy-MM-dd HH:mm:ss") +doAssert dstT1 == getLocalTime(toTime(dstT1)) +doAssert dstT2 == getLocalTime(toTime(dstT2)) + +# Comparison between Time objects should be detected by compiler +# as 'noSideEffect'. +proc cmpTimeNoSideEffect(t1: Time, t2: Time): bool {.noSideEffect.} = + result = t1 == t2 +doAssert cmpTimeNoSideEffect(0.fromSeconds, 0.fromSeconds) +# Additionally `==` generic for seq[T] has explicit 'noSideEffect' pragma +# so we can check above condition by comparing seq[Time] sequences +let seqA: seq[Time] = @[] +let seqB: seq[Time] = @[] +doAssert seqA == seqB + +for tz in [ + (0, "+0", "+00", "+00:00"), # UTC + (-3600, "+1", "+01", "+01:00"), # CET + (-39600, "+11", "+11", "+11:00"), # two digits + (-1800, "+0", "+00", "+00:30"), # half an hour + (7200, "-2", "-02", "-02:00"), # positive + (38700, "-10", "-10", "-10:45")]: # positive with three quaters hour + let ti = TimeInfo(monthday: 1, timezone: tz[0]) + doAssert ti.format("z") == tz[1] + doAssert ti.format("zz") == tz[2] + doAssert ti.format("zzz") == tz[3] + +block dstTest: + let nonDst = TimeInfo(year: 2015, month: mJan, monthday: 01, yearday: 0, + weekday: dThu, hour: 00, minute: 00, second: 00, isDST: false, timezone: 0) + var dst = nonDst + dst.isDst = true + # note that both isDST == true and isDST == false are valid here because + # DST is in effect on January 1st in some southern parts of Australia. + + doAssert nonDst.toTime() - dst.toTime() == 3600 + doAssert nonDst.format("z") == "+0" + doAssert dst.format("z") == "+1" + + # parsing will set isDST in relation to the local time. We take a date in + # January and one in July to maximize the probability to hit one date with DST + # and one without on the local machine. However, this is not guaranteed. + let + parsedJan = parse("2016-01-05 04:00:00+01:00", "yyyy-MM-dd HH:mm:sszzz") + parsedJul = parse("2016-07-01 04:00:00+01:00", "yyyy-MM-dd HH:mm:sszzz") + doAssert toTime(parsedJan) == fromSeconds(1451962800) + doAssert toTime(parsedJul) == fromSeconds(1467342000) diff --git a/tests/system/tdeepcopy.nim b/tests/system/tdeepcopy.nim new file mode 100644 index 000000000..5a582425a --- /dev/null +++ b/tests/system/tdeepcopy.nim @@ -0,0 +1,95 @@ +discard """ + output: "ok" + disabled: "true" +""" + +import tables, lists + +type + ListTable[K, V] = object + valList: DoublyLinkedList[V] + table: Table[K, DoublyLinkedNode[V]] + + ListTableRef*[K, V] = ref ListTable[K, V] + +proc initListTable*[K, V](initialSize = 64): ListTable[K, V] = + result.valList = initDoublyLinkedList[V]() + result.table = initTable[K, DoublyLinkedNode[V]]() + +proc newListTable*[K, V](initialSize = 64): ListTableRef[K, V] = + new(result) + result[] = initListTable[K, V](initialSize) + +proc `[]=`*[K, V](t: var ListTable[K, V], key: K, val: V) = + if key in t.table: + t.table[key].value = val + else: + let node = newDoublyLinkedNode(val) + t.valList.append(node) + t.table[key] = node + +proc `[]`*[K, V](t: ListTable[K, V], key: K): var V {.inline.} = + result = t.table[key].value + +proc len*[K, V](t: ListTable[K, V]): Natural {.inline.} = + result = t.table.len + +iterator values*[K, V](t: ListTable[K, V]): V = + for val in t.valList.items(): + yield val + +proc `[]=`*[K, V](t: ListTableRef[K, V], key: K, val: V) = + t[][key] = val + +proc `[]`*[K, V](t: ListTableRef[K, V], key: K): var V {.inline.} = + t[][key] + +proc len*[K, V](t: ListTableRef[K, V]): Natural {.inline.} = + t[].len + +iterator values*[K, V](t: ListTableRef[K, V]): V = + for val in t[].values: + yield val + +proc main() = + type SomeObj = ref object + + for outer in 0..10_000: + let myObj = new(SomeObj) + let table = newListTable[int, SomeObj]() + + table[0] = myObj + for i in 1..100: + table[i] = new(SomeObj) + + var myObj2: SomeObj + for val in table.values(): + if myObj2.isNil: + myObj2 = val + assert(myObj == myObj2) # passes + + var tableCopy: ListTableRef[int, SomeObj] + deepCopy(tableCopy, table) + + let myObjCopy = tableCopy[0] + var myObjCopy2: SomeObj = nil + for val in tableCopy.values(): + if myObjCopy2.isNil: + myObjCopy2 = val + + #echo cast[int](myObj) + #echo cast[int](myObjCopy) + #echo cast[int](myObjCopy2) + + assert(myObjCopy == myObjCopy2) # fails + + +type + PtrTable = object + counter, max: int + data: array[0..99, (pointer, pointer)] + +assert(sizeof(PtrTable) == 2*sizeof(int)+sizeof(pointer)*2*100) + +main() +echo "ok" diff --git a/tests/template/tconfusinglocal.nim b/tests/template/tconfusinglocal.nim new file mode 100644 index 000000000..9b2cdc954 --- /dev/null +++ b/tests/template/tconfusinglocal.nim @@ -0,0 +1,13 @@ + +# bug #4875 +type Bar = object + mFoo: int + +template foo(a: Bar): int = a.mFoo + +proc main = + let foo = 5 # Rename this to smth else to make it work + var b: Bar + echo b.foo + +main() diff --git a/tests/test_nimscript.nims b/tests/test_nimscript.nims index 436e990ef..2500bac73 100644 --- a/tests/test_nimscript.nims +++ b/tests/test_nimscript.nims @@ -14,7 +14,7 @@ import ospaths # import parseopt import parseutils # import pegs -import queues +import deques import sequtils import strutils import subexes diff --git a/tests/testament/categories.nim b/tests/testament/categories.nim index 3ed2f2196..809425653 100644 --- a/tests/testament/categories.nim +++ b/tests/testament/categories.nim @@ -115,6 +115,12 @@ proc dllTests(r: var TResults, cat: Category, options: string) = # ------------------------------ GC tests ------------------------------------- proc gcTests(r: var TResults, cat: Category, options: string) = + template testWithNone(filename: untyped) = + testSpec r, makeTest("tests/gc" / filename, options & + " --gc:none", cat, actionRun) + testSpec r, makeTest("tests/gc" / filename, options & + " -d:release --gc:none", cat, actionRun) + template testWithoutMs(filename: untyped) = testSpec r, makeTest("tests/gc" / filename, options, cat, actionRun) testSpec r, makeTest("tests/gc" / filename, options & @@ -144,6 +150,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) = test "gcleak" test "gcleak2" test "gctest" + testWithNone "gctest" test "gcleak3" test "gcleak4" # Disabled because it works and takes too long to run: @@ -375,8 +382,14 @@ proc `&.?`(a, b: string): string = proc `&?.`(a, b: string): string = # candidate for the stdlib? result = if a.endswith(b): a else: a & b + +proc processSingleTest(r: var TResults, cat: Category, options, test: string) = + let test = "tests" & DirSep &.? cat.string / test + + if existsFile(test): testSpec r, makeTest(test, options, cat) + else: echo "[Warning] - ", test, " test does not exist" -proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: string = "t*.nim") = +proc processCategory(r: var TResults, cat: Category, options: string) = case cat.string.normalize of "rodfiles": when false: compileRodFiles(r, cat, options) @@ -417,5 +430,5 @@ proc processCategory(r: var TResults, cat: Category, options: string, fileGlob: # We can't test it because it depends on a third party. discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. else: - for name in os.walkFiles("tests" & DirSep &.? cat.string / fileGlob): + for name in os.walkFiles("tests" & DirSep &.? cat.string / "t*.nim"): testSpec r, makeTest(name, options, cat) diff --git a/tests/testament/tester.nim b/tests/testament/tester.nim index 74ac58927..2734742f4 100644 --- a/tests/testament/tester.nim +++ b/tests/testament/tester.nim @@ -445,7 +445,7 @@ proc main() = let (dir, file) = splitPath(p.key.string) let (_, subdir) = splitPath(dir) var cat = Category(subdir) - processCategory(r, cat, p.cmdLineRest.string, file) + processSingleTest(r, cat, p.cmdLineRest.string, file) of "html": var commit = 0 discard parseInt(p.cmdLineRest.string, commit) diff --git a/tests/vm/tgorge.bat b/tests/vm/tgorge.bat new file mode 100644 index 000000000..24d365842 --- /dev/null +++ b/tests/vm/tgorge.bat @@ -0,0 +1 @@ +@echo gorge test \ No newline at end of file diff --git a/tests/vm/tgorge.nim b/tests/vm/tgorge.nim new file mode 100644 index 000000000..596f5d667 --- /dev/null +++ b/tests/vm/tgorge.nim @@ -0,0 +1,12 @@ +import os + +template getScriptDir(): string = + parentDir(instantiationInfo(-1, true).filename) + +const + execName = when defined(windows): "tgorge.bat" else: "sh tgorge.sh" + relOutput = gorge(execName) + absOutput = gorge(getScriptDir() / execName) + +doAssert relOutput == "gorge test" +doAssert absOutput == "gorge test" diff --git a/tests/vm/tgorge.sh b/tests/vm/tgorge.sh new file mode 100644 index 000000000..ba47afeae --- /dev/null +++ b/tests/vm/tgorge.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "gorge test" \ No newline at end of file diff --git a/tools/finish.nim b/tools/finish.nim new file mode 100644 index 000000000..cac001d79 --- /dev/null +++ b/tools/finish.nim @@ -0,0 +1,167 @@ + +# -------------- post unzip steps --------------------------------------------- + +import strutils, os, osproc + +when defined(windows): + import registry + + proc askBool(m: string): bool = + stdout.write m + while true: + let answer = stdin.readLine().normalize + case answer + of "y", "yes": + return true + of "n", "no": + return false + else: + echo "Please type 'y' or 'n'" + + proc askNumber(m: string; a, b: int): int = + stdout.write m + stdout.write " [" & $a & ".." & $b & "] " + while true: + let answer = stdin.readLine() + try: + result = parseInt answer + if result < a or result > b: + raise newException(ValueError, "number out of range") + break + except ValueError: + echo "Please type in a number between ", a, " and ", b + + proc patchConfig(mingw: string) = + const + cfgFile = "config/nim.cfg" + lookFor = """#gcc.path = r"$nim\dist\mingw\bin"""" + replacePattern = """gcc.path = r"$1"""" + try: + let cfg = readFile(cfgFile) + let newCfg = cfg.replace(lookFor, replacePattern % mingw) + if newCfg == cfg: + echo "Could not patch 'config/nim.cfg' [Error]" + echo "Reason: patch substring not found:" + echo lookFor + else: + writeFile(cfgFile, newCfg) + except IOError: + echo "Could not access 'config/nim.cfg' [Error]" + + proc addToPathEnv(e: string) = + let p = getUnicodeValue(r"Environment", "Path", HKEY_CURRENT_USER) + let x = if e.contains(Whitespace): "\"" & e & "\"" else: e + setUnicodeValue(r"Environment", "Path", p & ";" & x, HKEY_CURRENT_USER) + + proc createShortcut(src, dest: string; icon = "") = + var cmd = "bin\\makelink.exe \"" & src & "\" \"\" \"" & dest & + ".lnk\" \"\" 1 \"" & splitFile(src).dir & "\"" + if icon.len != 0: + cmd.add " \"" & icon & "\" 0" + discard execShellCmd(cmd) + + proc createStartMenuEntry() = + let appdata = getEnv("APPDATA") + if appdata.len == 0: return + let dest = appdata & r"\Microsoft\Windows\Start Menu\Programs\Nim-" & + NimVersion + if dirExists(dest): return + if askBool("Would like to add Nim-" & NimVersion & + " to your start menu? (y/n) "): + createDir(dest) + createShortcut(getCurrentDir() / "tools" / "start.bat", dest / "Nim", + getCurrentDir() / r"icons\nim.ico") + if fileExists("doc/overview.html"): + createShortcut(getCurrentDir() / "doc" / "html" / "overview.html", + dest / "Overview") + if dirExists(r"dist\aporia-0.4.0"): + createShortcut(getCurrentDir() / r"dist\aporia-0.4.0\bin\aporia.exe", + dest / "Aporia") + + proc checkGccArch(mingw: string): bool = + let gccExe = mingw / r"gcc.exe" + if fileExists(gccExe): + try: + let arch = execProcess(gccExe, ["-dumpmachine"], nil, {poStdErrToStdOut, + poUsePath}) + when hostCPU == "i386": + result = arch.startsWith("i686-") + elif hostCPU == "amd64": + result = arch.startsWith("x86_64-") + else: + {.error: "Unknown CPU for Windows.".} + except OSError, IOError: + result = false + + proc tryDirs(incompat: var seq[string]; dirs: varargs[string]): string = + let bits = $(sizeof(pointer)*8) + for d in dirs: + if dirExists d: + let x = expandFilename(d / "bin") + if checkGccArch(x): return x + else: incompat.add x + elif dirExists(d & bits): + let x = expandFilename((d & bits) / "bin") + if checkGccArch(x): return x + else: incompat.add x + +proc main() = + when defined(windows): + let desiredPath = expandFilename(getCurrentDir() / "bin") + let p = getUnicodeValue(r"Environment", "Path", + HKEY_CURRENT_USER) + var alreadyInPath = false + var mingWchoices: seq[string] = @[] + var incompat: seq[string] = @[] + for x in p.split(';'): + if x.len == 0: continue + let y = try: expandFilename(if x[0] == '"' and x[^1] == '"': + substr(x, 1, x.len-2) else: x) + except: "" + if y == desiredPath: alreadyInPath = true + if y.toLowerAscii.contains("mingw"): + if dirExists(y): + if checkGccArch(y): mingWchoices.add y + else: incompat.add y + + if alreadyInPath: + echo "bin/nim.exe is already in your PATH [Skipping]" + else: + if askBool("nim.exe is not in your PATH environment variable.\n" & + "Should it be added permanently? (y/n) "): + addToPathEnv(desiredPath) + if mingWchoices.len == 0: + # No mingw in path, so try a few locations: + let alternative = tryDirs(incompat, "dist/mingw", "../mingw", r"C:\mingw") + if alternative.len == 0: + if incompat.len > 0: + echo "The following *incompatible* MingW installations exist" + for x in incompat: echo x + echo "No compatible MingW candidates found " & + "in the standard locations [Error]" + else: + if askBool("Found a MingW directory that is not in your PATH.\n" & + alternative & + "\nShould it be added to your PATH permanently? (y/n) "): + addToPathEnv(alternative) + elif askBool("Do you want to patch Nim's config to use this? (y/n) "): + patchConfig(alternative) + elif mingWchoices.len == 1: + if askBool("MingW installation found at " & mingWchoices[0] & "\n" & + "Do you want to patch Nim's config to use this?\n" & + "(Not required since it's in your PATH!) (y/n) "): + patchConfig(mingWchoices[0]) + else: + echo "Multiple MingW installations found: " + for i in 0..high(mingWchoices): + echo "[", i, "] ", mingWchoices[i] + if askBool("Do you want to patch Nim's config to use one of these? (y/n) "): + let idx = askNumber("Which one do you want to use for Nim? ", + 1, len(mingWchoices)) + patchConfig(mingWchoices[idx-1]) + createStartMenuEntry() + else: + echo("Add ", getCurrentDir(), "/bin to your PATH...") + +when isMainModule: + main() diff --git a/tools/niminst/buildbat.tmpl b/tools/niminst/buildbat.tmpl index 2a8da144d..278b6caea 100644 --- a/tools/niminst/buildbat.tmpl +++ b/tools/niminst/buildbat.tmpl @@ -1,5 +1,5 @@ #? stdtmpl(subsChar='?') | standard -#proc generateBuildBatchScript(c: ConfigData, winIndex, cpuIndex: int): string = +#proc generateBuildBatchScript(c: ConfigData, winIndex, cpuIndex: int): string = # result = "@echo off\nREM Generated by niminst\n" SET CC=gcc SET LINKER=gcc @@ -23,8 +23,8 @@ CALL %CC% %COMP_FLAGS% -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} IF ERRORLEVEL 1 (GOTO:END) # end for -ECHO %LINKER% -o ?{"%BIN_DIR%"\toLower(c.name)}.exe ?linkCmd %LINK_FLAGS% -CALL %LINKER% -o ?{"%BIN_DIR%"\toLower(c.name)}.exe ?linkCmd %LINK_FLAGS% +ECHO %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% +CALL %LINKER% -o ?{"%BIN_DIR%"\toLowerAscii(c.name)}.exe ?linkCmd %LINK_FLAGS% # end block diff --git a/tools/niminst/buildsh.tmpl b/tools/niminst/buildsh.tmpl index 220ecdb7f..c571f4d66 100644 --- a/tools/niminst/buildsh.tmpl +++ b/tools/niminst/buildsh.tmpl @@ -53,7 +53,7 @@ uos=`echo $uos | tr "[:upper:]" "[:lower:]"` case $uos in *linux* ) myos="linux" - LINK_FLAGS="$LINK_FLAGS -ldl -lm" + LINK_FLAGS="$LINK_FLAGS -ldl -lm -lrt" ;; *dragonfly* ) myos="freebsd" @@ -94,6 +94,9 @@ case $uos in myos="haiku" LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" ;; + *mingw* ) + myos="windows" + ;; *) echo 2>&1 "Error: unknown operating system: $uos" exit 1 @@ -144,7 +147,7 @@ case $myos in $CC $COMP_FLAGS -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} # add(linkCmd, " \\\n" & changeFileExt(f, "o")) # end for - $LINKER -o ?{"$binDir/" & toLower(c.name)} ?linkCmd $LINK_FLAGS + $LINKER -o ?{"$binDir/" & toLowerAscii(c.name)} ?linkCmd $LINK_FLAGS ;; # end for *) diff --git a/tools/niminst/debcreation.nim b/tools/niminst/debcreation.nim index 36b2a29ec..0ecea132f 100644 --- a/tools/niminst/debcreation.nim +++ b/tools/niminst/debcreation.nim @@ -148,7 +148,7 @@ proc prepDeb*(packName, version, mtnName, mtnEmail, shortDesc, desc: string, ## binaries/config/docs/lib: files relative to nim's root, that need to ## be installed. - let pkgName = packName.toLower() + let pkgName = packName.toLowerAscii() var workingDir = getTempDir() / "niminst" / "deb" var upstreamSource = (pkgName & "-" & version) @@ -168,7 +168,7 @@ proc prepDeb*(packName, version, mtnName, mtnEmail, shortDesc, desc: string, echo("Creating necessary files in debian/") createDir(workingDir / upstreamSource / "debian") - template writeDebian(f, s: string): expr = + template writeDebian(f, s: string) = writeFile(workingDir / upstreamSource / "debian" / f, s) var controlFile = createControl(pkgName, makeMtn(mtnName, mtnEmail), diff --git a/tools/niminst/deinstall.tmpl b/tools/niminst/deinstall.tmpl index 3cdfbf45d..bba310f76 100644 --- a/tools/niminst/deinstall.tmpl +++ b/tools/niminst/deinstall.tmpl @@ -1,7 +1,7 @@ #? stdtmpl(subsChar='?') | standard #proc generateDeinstallScript(c: ConfigData): string = # result = "#! /bin/sh\n# Generated by niminst\n" -# var proj = c.name.toLower +# var proj = c.name.toLowerAscii if [ $# -eq 1 ] ; then case $1 in diff --git a/tools/niminst/install.tmpl b/tools/niminst/install.tmpl index 3f17840a8..d72b132ef 100644 --- a/tools/niminst/install.tmpl +++ b/tools/niminst/install.tmpl @@ -1,7 +1,7 @@ #? stdtmpl(subsChar = '?') | standard #proc generateInstallScript(c: ConfigData): string = # result = "#! /bin/sh\n# Generated by niminst\n" -# var proj = c.name.toLower +# var proj = c.name.toLowerAscii ## Current directory you start script from BASE_DIR=$(pwd) diff --git a/tools/niminst/makefile.tmpl b/tools/niminst/makefile.tmpl index 5c95ccda9..ce2db1c48 100644 --- a/tools/niminst/makefile.tmpl +++ b/tools/niminst/makefile.tmpl @@ -157,7 +157,7 @@ endif %.o: %.c $(CC) $(COMP_FLAGS) -Ic_code -c $< -o $@ -?{"$(binDir)/" & toLower(c.name)}: $(oFiles) +?{"$(binDir)/" & toLowerAscii(c.name)}: $(oFiles) @mkdir -p $(binDir) $(LINKER) -o $@ $^ $(LINK_FLAGS) @echo "SUCCESS" @@ -165,4 +165,4 @@ endif .PHONY: clean clean: - rm -f $(oFiles) ?{"$(binDir)/" & toLower(c.name)} + rm -f $(oFiles) ?{"$(binDir)/" & toLowerAscii(c.name)} diff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index b63849a10..e0b8ad9b3 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -313,7 +313,7 @@ proc parseIniFile(c: var ConfigData) = of cfgSectionStart: section = normalize(k.section) of cfgKeyValuePair: - var v = k.value % c.vars + var v = `%`(k.value, c.vars, {useEnvironment, useEmpty}) c.vars[k.key] = v case section @@ -552,7 +552,7 @@ proc srcdist(c: var ConfigData) = # --------------------- generate inno setup ----------------------------------- proc setupDist(c: var ConfigData) = let scrpt = generateInnoSetup(c) - let n = "build" / "install_$#_$#.iss" % [toLower(c.name), c.version] + let n = "build" / "install_$#_$#.iss" % [toLowerAscii(c.name), c.version] writeFile(n, scrpt, "\13\10") when defined(windows): if c.innosetup.path.len == 0: @@ -569,7 +569,7 @@ proc setupDist(c: var ConfigData) = # --------------------- generate NSIS setup ----------------------------------- proc setupDist2(c: var ConfigData) = let scrpt = generateNsisSetup(c) - let n = "build" / "install_$#_$#.nsi" % [toLower(c.name), c.version] + let n = "build" / "install_$#_$#.nsi" % [toLowerAscii(c.name), c.version] writeFile(n, scrpt, "\13\10") when defined(windows): if c.nsisSetup.path.len == 0: @@ -586,7 +586,7 @@ proc setupDist2(c: var ConfigData) = # ------------------ generate ZIP file --------------------------------------- when haveZipLib: proc zipDist(c: var ConfigData) = - var proj = toLower(c.name) & "-" & c.version + var proj = toLowerAscii(c.name) & "-" & c.version var n = "$#.zip" % proj if c.outdir.len == 0: n = "build" / n else: n = c.outdir / n @@ -618,40 +618,49 @@ when haveZipLib: quit("Cannot open for writing: " & n) proc xzDist(c: var ConfigData; windowsZip=false) = - let proj = toLower(c.name) & "-" & c.version + let proj = toLowerAscii(c.name) & "-" & c.version let tmpDir = if c.outdir.len == 0: "build" else: c.outdir - template processFile(z, dest, src) = - let s = src - let d = dest - echo "Copying ", s, " to ", tmpDir / d - let destdir = tmpdir / d.splitFile.dir - if not dirExists(destdir): createDir(destdir) - copyFileWithPermissions(s, tmpDir / d) - - processFile(z, proj / buildBatFile32, "build" / buildBatFile32) - processFile(z, proj / buildBatFile64, "build" / buildBatFile64) - processFile(z, proj / buildShFile, "build" / buildShFile) - processFile(z, proj / makeFile, "build" / makeFile) - processFile(z, proj / installShFile, installShFile) - processFile(z, proj / deinstallShFile, deinstallShFile) + proc processFile(destFile, src: string) = + let dest = tmpDir / destFile + echo "Copying ", src, " to ", dest + if not existsFile(src): + echo "[Warning] Source file doesn't exist: ", src + let destDir = dest.splitFile.dir + if not dirExists(destDir): createDir(destDir) + copyFileWithPermissions(src, dest) + + if not windowsZip and not existsFile("build" / buildBatFile32): + quit("No C sources found in ./build/, please build by running " & + "./koch csource -d:release.") + if not windowsZip: + processFile(proj / buildBatFile32, "build" / buildBatFile32) + processFile(proj / buildBatFile64, "build" / buildBatFile64) + processFile(proj / buildShFile, "build" / buildShFile) + processFile(proj / makeFile, "build" / makeFile) + processFile(proj / installShFile, installShFile) + processFile(proj / deinstallShFile, deinstallShFile) for f in walkFiles(c.libpath / "lib/*.h"): - processFile(z, proj / "c_code" / extractFilename(f), f) + processFile(proj / "c_code" / extractFilename(f), f) for osA in 1..c.oses.len: for cpuA in 1..c.cpus.len: var dir = buildDir(osA, cpuA) for k, f in walkDir("build" / dir): - if k == pcFile: processFile(z, proj / dir / extractFilename(f), f) + if k == pcFile: processFile(proj / dir / extractFilename(f), f) + else: + for f in items(c.cat[fcWinBin]): + let filename = f.extractFilename + processFile(proj / "bin" / filename, f) let osSpecific = if windowsZip: fcWindows else: fcUnix for cat in items({fcConfig..fcOther, osSpecific, fcNimble}): echo("Current category: ", cat) - for f in items(c.cat[cat]): processFile(z, proj / f, f) + for f in items(c.cat[cat]): processFile(proj / f, f) # Copy the .nimble file over let nimbleFile = c.nimblePkgName & ".nimble" - processFile(z, proj / nimbleFile, nimbleFile) + processFile(proj / nimbleFile, nimbleFile) when true: let oldDir = getCurrentDir() @@ -683,11 +692,11 @@ proc debDist(c: var ConfigData) = echo("Copying source to tmp/niminst/deb/") var currentSource = getCurrentDir() var workingDir = getTempDir() / "niminst" / "deb" - var upstreamSource = (c.name.toLower() & "-" & c.version) + var upstreamSource = (c.name.toLowerAscii() & "-" & c.version) createDir(workingDir / upstreamSource) - template copyNimDist(f, dest: string): stmt = + template copyNimDist(f, dest: string) = createDir((workingDir / upstreamSource / dest).splitFile.dir) copyFile(currentSource / f, workingDir / upstreamSource / dest) diff --git a/tools/niminst/nsis.tmpl b/tools/niminst/nsis.tmpl index 639a01b6b..95d4652e3 100644 --- a/tools/niminst/nsis.tmpl +++ b/tools/niminst/nsis.tmpl @@ -202,7 +202,7 @@ ; Shortcuts # if d.len >= 6: # let startMenuEntry = d[5] - # let e = splitFile(startMenuEntry).name.capitalize + # let e = splitFile(startMenuEntry).name.capitalizeAscii !insertmacro MUI_STARTMENU_WRITE_BEGIN Application CreateShortCut "$SMPROGRAMS\$ICONS_GROUP\?{e}.lnk" "$INSTDIR\?dir\?{startMenuEntry.toWin}" !insertmacro MUI_STARTMENU_WRITE_END diff --git a/tools/nimsuggest/nimsuggest.nim b/tools/nimsuggest/nimsuggest.nim new file mode 100644 index 000000000..b5e7b282f --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nim @@ -0,0 +1,475 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nimsuggest is a tool that helps to give editors IDE like capabilities. + +import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp +# Do NOT import suggest. It will lead to wierd bugs with +# suggestionResultHook, because suggest.nim is included by sigmatch. +# So we import that one instead. +import compiler/options, compiler/commands, compiler/modules, compiler/sem, + compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf, + compiler/extccomp, compiler/condsyms, compiler/lists, + compiler/sigmatch, compiler/ast, compiler/scriptconfig, + compiler/idents, compiler/modulegraphs + +when defined(windows): + import winlean +else: + import posix + +const DummyEof = "!EOF!" +const Usage = """ +Nimsuggest - Tool to give every editor IDE like capabilities for Nim +Usage: + nimsuggest [options] projectfile.nim + +Options: + --port:PORT port, by default 6000 + --address:HOST binds to that address, by default "" + --stdin read commands from stdin and write results to + stdout instead of using sockets + --epc use emacs epc mode + --debug enable debug output + --log enable verbose logging to nimsuggest.log file + --v2 use version 2 of the protocol; more features and + much faster + --tester implies --v2 and --stdin and outputs a line + '""" & DummyEof & """' for the tester + +The server then listens to the connection and takes line-based commands. + +In addition, all command line options of Nim that do not affect code generation +are supported. +""" +type + Mode = enum mstdin, mtcp, mepc + +var + gPort = 6000.Port + gAddress = "" + gMode: Mode + gEmitEof: bool # whether we write '!EOF!' dummy lines + gLogging = false + +const + seps = {':', ';', ' ', '\t'} + Help = "usage: sug|con|def|use|dus|chk|mod|highlight|outline file.nim[;dirtyfile.nim]:line:col\n" & + "type 'quit' to quit\n" & + "type 'debug' to toggle debug mode on/off\n" & + "type 'terse' to toggle terse mode on/off" + +type + EUnexpectedCommand = object of Exception + +proc logStr(line: string) = + var f: File + if open(f, getHomeDir() / "nimsuggest.log", fmAppend): + f.writeLine(line) + f.close() + +proc parseQuoted(cmd: string; outp: var string; start: int): int = + var i = start + i += skipWhitespace(cmd, i) + if cmd[i] == '"': + i += parseUntil(cmd, outp, '"', i+1)+2 + else: + i += parseUntil(cmd, outp, seps, i) + result = i + +proc sexp(s: IdeCmd|TSymKind): SexpNode = sexp($s) + +proc sexp(s: Suggest): SexpNode = + # If you change the order here, make sure to change it over in + # nim-mode.el too. + result = convertSexp([ + s.section, + s.symkind, + s.qualifiedPath.map(newSString), + s.filePath, + s.forth, + s.line, + s.column, + s.doc + ]) + +proc sexp(s: seq[Suggest]): SexpNode = + result = newSList() + for sug in s: + result.add(sexp(sug)) + +proc listEpc(): SexpNode = + # This function is called from Emacs to show available options. + let + argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol)) + docstring = sexp("line starts at 1, column at 0, dirtyfile is optional") + result = newSList() + for command in ["sug", "con", "def", "use", "dus", "chk", "mod"]: + let + cmd = sexp(command) + methodDesc = newSList() + methodDesc.add(cmd) + methodDesc.add(argspecs) + methodDesc.add(docstring) + result.add(methodDesc) + +proc findNode(n: PNode): PSym = + #echo "checking node ", n.info + if n.kind == nkSym: + if isTracked(n.info, n.sym.name.s.len): return n.sym + else: + for i in 0 ..< safeLen(n): + let res = n.sons[i].findNode + if res != nil: return res + +proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym = + let m = graph.getModule(gTrackPos.fileIndex) + #echo m.isNil, " I knew it ", gTrackPos.fileIndex + if m != nil and m.ast != nil: + result = m.ast.findNode + +proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int; + graph: ModuleGraph; cache: IdentCache) = + if gLogging: + logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]") + gIdeCmd = cmd + if cmd == ideUse and suggestVersion != 2: + graph.resetAllModules() + var isKnownFile = true + let dirtyIdx = file.fileInfoIdx(isKnownFile) + + if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile) + else: msgs.setDirtyFile(dirtyIdx, nil) + + gTrackPos = newLineInfo(dirtyIdx, line, col) + gErrorCounter = 0 + if suggestVersion < 2: + usageSym = nil + if not isKnownFile: + graph.compileProject(cache) + if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and + dirtyfile.len == 0: + discard "no need to recompile anything" + else: + let modIdx = graph.parentModule(dirtyIdx) + graph.markDirty dirtyIdx + graph.markClientsDirty dirtyIdx + if gIdeCmd != ideMod: + graph.compileProject(cache, modIdx) + if gIdeCmd in {ideUse, ideDus}: + let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: usageSym + if u != nil: + listUsages(u) + else: + localError(gTrackPos, "found no symbol at this position " & $gTrackPos) + +proc executeEpc(cmd: IdeCmd, args: SexpNode; + graph: ModuleGraph; cache: IdentCache) = + let + file = args[0].getStr + line = args[1].getNum + column = args[2].getNum + var dirtyfile = "" + if len(args) > 3: + dirtyfile = args[3].getStr(nil) + execute(cmd, file, dirtyfile, int(line), int(column), graph, cache) + +proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string, + return_symbol = "return") = + let response = $convertSexp([newSSymbol(return_symbol), uid, s]) + socket.send(toHex(len(response), 6)) + socket.send(response) + +template sendEpc(results: typed, tdef, hook: untyped) = + hook = proc (s: tdef) = + results.add( + # Put newlines to parse output by flycheck-nim.el + when results is string: s & "\n" + else: s + ) + + executeEpc(gIdeCmd, args, graph, cache) + let res = sexp(results) + if gLogging: + logStr($res) + returnEPC(client, uid, res) + +template checkSanity(client, sizeHex, size, messageBuffer: typed) = + if client.recv(sizeHex, 6) != 6: + raise newException(ValueError, "didn't get all the hexbytes") + if parseHex(sizeHex, size) == 0: + raise newException(ValueError, "invalid size hex: " & $sizeHex) + if client.recv(messageBuffer, size) != size: + raise newException(ValueError, "didn't get all the bytes") + +template setVerbosity(level: typed) = + gVerbosity = level + gNotes = NotesVerbosity[gVerbosity] + +proc connectToNextFreePort(server: Socket, host: string): Port = + server.bindaddr(Port(0), host) + let (_, port) = server.getLocalAddr + result = port + +proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) = + template toggle(sw) = + if sw in gGlobalOptions: + excl(gGlobalOptions, sw) + else: + incl(gGlobalOptions, sw) + return + + template err() = + echo Help + return + + var opc = "" + var i = parseIdent(cmd, opc, 0) + case opc.normalize + of "sug": gIdeCmd = ideSug + of "con": gIdeCmd = ideCon + of "def": gIdeCmd = ideDef + of "use": gIdeCmd = ideUse + of "dus": gIdeCmd = ideDus + of "mod": gIdeCmd = ideMod + of "chk": + gIdeCmd = ideChk + incl(gGlobalOptions, optIdeDebug) + of "highlight": gIdeCmd = ideHighlight + of "outline": gIdeCmd = ideOutline + of "quit": quit() + of "debug": toggle optIdeDebug + of "terse": toggle optIdeTerse + else: err() + var dirtyfile = "" + var orig = "" + i = parseQuoted(cmd, orig, i) + if cmd[i] == ';': + i = parseQuoted(cmd, dirtyfile, i+1) + i += skipWhile(cmd, seps, i) + var line = -1 + var col = 0 + i += parseInt(cmd, line, i) + i += skipWhile(cmd, seps, i) + i += parseInt(cmd, col, i) + + execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache) + +proc serveStdin(graph: ModuleGraph; cache: IdentCache) = + if gEmitEof: + echo DummyEof + while true: + let line = readLine(stdin) + parseCmdLine line, graph, cache + echo DummyEof + flushFile(stdout) + else: + echo Help + var line = "" + while readLineFromStdin("> ", line): + parseCmdLine line, graph, cache + echo "" + flushFile(stdout) + +proc serveTcp(graph: ModuleGraph; cache: IdentCache) = + var server = newSocket() + server.bindAddr(gPort, gAddress) + var inp = "".TaintedString + server.listen() + + while true: + var stdoutSocket = newSocket() + msgs.writelnHook = proc (line: string) = + stdoutSocket.send(line & "\c\L") + + accept(server, stdoutSocket) + + stdoutSocket.readLine(inp) + parseCmdLine inp.string, graph, cache + + stdoutSocket.send("\c\L") + stdoutSocket.close() + +proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) = + var client = newSocket() + # Wait for connection + accept(server, client) + if gLogging: + var it = searchPaths.head + while it != nil: + logStr(PStrEntry(it).data) + it = it.next + msgs.writelnHook = proc (line: string) = logStr(line) + + while true: + var + sizeHex = "" + size = 0 + messageBuffer = "" + checkSanity(client, sizeHex, size, messageBuffer) + let + message = parseSexp($messageBuffer) + epcAPI = message[0].getSymbol + case epcAPI: + of "call": + let + uid = message[1].getNum + args = message[3] + + gIdeCmd = parseIdeCmd(message[2].getSymbol) + case gIdeCmd + of ideChk: + setVerbosity(1) + # Use full path because other emacs plugins depends it + gListFullPaths = true + incl(gGlobalOptions, optIdeDebug) + var hints_or_errors = "" + sendEpc(hints_or_errors, string, msgs.writelnHook) + of ideSug, ideCon, ideDef, ideUse, ideDus, ideOutline, ideHighlight: + setVerbosity(0) + var suggests: seq[Suggest] = @[] + sendEpc(suggests, Suggest, suggestionResultHook) + else: discard + of "methods": + returnEpc(client, message[1].getNum, listEPC()) + of "epc-error": + stderr.writeline("recieved epc error: " & $messageBuffer) + raise newException(IOError, "epc error") + else: + let errMessage = case epcAPI + of "return", "return-error": + "no return expected" + else: + "unexpected call: " & epcAPI + raise newException(EUnexpectedCommand, errMessage) + +proc mainCommand(graph: ModuleGraph; cache: IdentCache) = + clearPasses() + registerPass verbosePass + registerPass semPass + gCmd = cmdIdeTools + incl gGlobalOptions, optCaasEnabled + isServing = true + wantMainModule() + appendStr(searchPaths, options.libpath) + #if gProjectFull.len != 0: + # current path is always looked first for modules + # prependStr(searchPaths, gProjectPath) + + # do not stop after the first error: + msgs.gErrorMax = high(int) + + case gMode + of mstdin: + compileProject(graph, cache) + #modules.gFuzzyGraphChecking = false + serveStdin(graph, cache) + of mtcp: + # until somebody accepted the connection, produce no output (logging is too + # slow for big projects): + msgs.writelnHook = proc (msg: string) = discard + compileProject(graph, cache) + #modules.gFuzzyGraphChecking = false + serveTcp(graph, cache) + of mepc: + var server = newSocket() + let port = connectToNextFreePort(server, "localhost") + server.listen() + echo port + compileProject(graph, cache) + serveEpc(server, graph, cache) + +proc processCmdLine*(pass: TCmdLinePass, cmd: string) = + var p = parseopt.initOptParser(cmd) + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongoption, cmdShortOption: + case p.key.normalize + of "port": + gPort = parseInt(p.val).Port + gMode = mtcp + of "address": + gAddress = p.val + gMode = mtcp + of "stdin": gMode = mstdin + of "epc": + gMode = mepc + gVerbosity = 0 # Port number gotta be first. + of "debug": + incl(gGlobalOptions, optIdeDebug) + of "v2": + suggestVersion = 2 + of "tester": + suggestVersion = 2 + gMode = mstdin + gEmitEof = true + of "log": + gLogging = true + else: processSwitch(pass, p) + of cmdArgument: + options.gProjectName = unixToNativePath(p.key) + # if processArgument(pass, p, argsCount): break + +proc handleCmdLine(cache: IdentCache) = + if paramCount() == 0: + stdout.writeline(Usage) + else: + processCmdLine(passCmd1, "") + if gMode != mstdin: + msgs.writelnHook = proc (msg: string) = discard + if gProjectName != "": + try: + gProjectFull = canonicalizePath(gProjectName) + except OSError: + gProjectFull = gProjectName + var p = splitFile(gProjectFull) + gProjectPath = canonicalizePath p.dir + gProjectName = p.name + else: + gProjectPath = canonicalizePath getCurrentDir() + + # Find Nim's prefix dir. + let binaryPath = findExe("nim") + if binaryPath == "": + raise newException(IOError, + "Cannot find Nim standard library: Nim compiler not in PATH") + gPrefixDir = binaryPath.splitPath().head.parentDir() + #msgs.writelnHook = proc (line: string) = logStr(line) + + loadConfigs(DefaultConfig, cache) # load all config files + # now process command line arguments again, because some options in the + # command line can overwite the config file's settings + options.command = "nimsuggest" + let scriptFile = gProjectFull.changeFileExt("nims") + if fileExists(scriptFile): + runNimScript(cache, scriptFile, freshDefines=false) + # 'nim foo.nims' means to just run the NimScript file and do nothing more: + if scriptFile == gProjectFull: return + elif fileExists(gProjectPath / "config.nims"): + # directory wide NimScript file + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) + + extccomp.initVars() + processCmdLine(passCmd2, "") + + let graph = newModuleGraph() + graph.suggestMode = true + mainCommand(graph, cache) + +when false: + proc quitCalled() {.noconv.} = + writeStackTrace() + + addQuitProc(quitCalled) + +condsyms.initDefines() +defineSymbol "nimsuggest" +handleCmdline(newIdentCache()) diff --git a/tools/nimsuggest/nimsuggest.nim.cfg b/tools/nimsuggest/nimsuggest.nim.cfg new file mode 100644 index 000000000..949bd18e8 --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nim.cfg @@ -0,0 +1,16 @@ +# Special configuration file for the Nim project + +gc:markAndSweep + +hint[XDeclaredButNotUsed]:off + +path:"$lib/packages/docutils" + +define:useStdoutAsStdmsg +define:nimsuggest + +#cs:partial +#define:useNodeIds +#define:booting +#define:noDocgen +--path:"$nim" diff --git a/tools/nimsuggest/nimsuggest.nimble b/tools/nimsuggest/nimsuggest.nimble new file mode 100644 index 000000000..3651e12bd --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nimble @@ -0,0 +1,11 @@ +[Package] +name = "nimsuggest" +version = "0.1.0" +author = "Andreas Rumpf" +description = "Tool for providing auto completion data for Nim source code." +license = "MIT" + +bin = "nimsuggest" + +[Deps] +Requires: "nim >= 0.11.2, compiler#head" diff --git a/tools/nimsuggest/sexp.nim b/tools/nimsuggest/sexp.nim new file mode 100644 index 000000000..cf08111d7 --- /dev/null +++ b/tools/nimsuggest/sexp.nim @@ -0,0 +1,697 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf, Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import + hashes, strutils, lexbase, streams, unicode, macros + +type + SexpEventKind* = enum ## enumeration of all events that may occur when parsing + sexpError, ## an error occurred during parsing + sexpEof, ## end of file reached + sexpString, ## a string literal + sexpSymbol, ## a symbol + sexpInt, ## an integer literal + sexpFloat, ## a float literal + sexpNil, ## the value ``nil`` + sexpDot, ## the dot to separate car/cdr + sexpListStart, ## start of a list: the ``(`` token + sexpListEnd, ## end of a list: the ``)`` token + + TTokKind = enum # must be synchronized with SexpEventKind! + tkError, + tkEof, + tkString, + tkSymbol, + tkInt, + tkFloat, + tkNil, + tkDot, + tkParensLe, + tkParensRi + tkSpace + + SexpError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errParensRiExpected, ## ``)`` expected + errQuoteExpected, ## ``"`` expected + errEofExpected, ## EOF expected + + SexpParser* = object of BaseLexer ## the parser object. + a: string + tok: TTokKind + kind: SexpEventKind + err: SexpError + +const + errorMessages: array[SexpError, string] = [ + "no error", + "invalid token", + "')' expected", + "'\"' or \"'\" expected", + "EOF expected", + ] + tokToStr: array[TTokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "symbol", + "int literal", + "float literal", + "nil", + ".", + "(", ")", "space" + ] + +proc close*(my: var SexpParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: SexpParser): string {.inline.} = + ## returns the character data for the events: ``sexpInt``, ``sexpFloat``, + ## ``sexpString`` + assert(my.kind in {sexpInt, sexpFloat, sexpString}) + result = my.a + +proc getInt*(my: SexpParser): BiggestInt {.inline.} = + ## returns the number for the event: ``sexpInt`` + assert(my.kind == sexpInt) + result = parseBiggestInt(my.a) + +proc getFloat*(my: SexpParser): float {.inline.} = + ## returns the number for the event: ``sexpFloat`` + assert(my.kind == sexpFloat) + result = parseFloat(my.a) + +proc kind*(my: SexpParser): SexpEventKind {.inline.} = + ## returns the current event type for the SEXP parser + result = my.kind + +proc getColumn*(my: SexpParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: SexpParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc errorMsg*(my: SexpParser): string = + ## returns a helpful error message for the event ``sexpError`` + assert(my.kind == sexpError) + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: SexpParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), e & " expected"] + +proc handleHexChar(c: char, x: var int): bool = + result = true # Success + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: result = false # error + +proc parseString(my: var SexpParser): TTokKind = + result = tkString + var pos = my.bufpos + 1 + var buf = my.buf + while true: + case buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + inc(pos) + break + of '\\': + case buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'u': + inc(pos, 2) + var r: int + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + add(my.a, '\L') + else: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc parseNumber(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] == '-': + add(my.a, '-') + inc(pos) + if buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'E', 'e'}: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'+', '-'}: + add(my.a, buf[pos]) + inc(pos) + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseSymbol(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] in IdentStartChars: + while buf[pos] in IdentChars: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok(my: var SexpParser): TTokKind = + setLen(my.a, 0) + case my.buf[my.bufpos] + of '-', '0'..'9': # numbers that start with a . are not parsed + # correctly. + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': #" # gotta fix nim-mode + result = parseString(my) + of '(': + inc(my.bufpos) + result = tkParensLe + of ')': + inc(my.bufpos) + result = tkParensRi + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseSymbol(my) + if my.a == "nil": + result = tkNil + else: + result = tkSymbol + of ' ': + result = tkSpace + inc(my.bufpos) + of '.': + result = tkDot + inc(my.bufpos) + else: + inc(my.bufpos) + result = tkError + my.tok = result + +# ------------- higher level interface --------------------------------------- + +type + SexpNodeKind* = enum ## possible SEXP node types + SNil, + SInt, + SFloat, + SString, + SSymbol, + SList, + SCons + + SexpNode* = ref SexpNodeObj ## SEXP node + SexpNodeObj* {.acyclic.} = object + case kind*: SexpNodeKind + of SString: + str*: string + of SSymbol: + symbol*: string + of SInt: + num*: BiggestInt + of SFloat: + fnum*: float + of SList: + elems*: seq[SexpNode] + of SCons: + car: SexpNode + cdr: SexpNode + of SNil: + discard + + Cons = tuple[car: SexpNode, cdr: SexpNode] + + SexpParsingError* = object of ValueError ## is raised for a SEXP error + +proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} = + ## raises an `ESexpParsingError` exception. + raise newException(SexpParsingError, errorMsgExpected(p, msg)) + +proc newSString*(s: string): SexpNode {.procvar.}= + ## Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc newSStringMove(s: string): SexpNode = + new(result) + result.kind = SString + shallowCopy(result.str, s) + +proc newSInt*(n: BiggestInt): SexpNode {.procvar.} = + ## Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc newSFloat*(n: float): SexpNode {.procvar.} = + ## Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc newSNil*(): SexpNode {.procvar.} = + ## Creates a new `SNil SexpNode`. + new(result) + +proc newSCons*(car, cdr: SexpNode): SexpNode {.procvar.} = + ## Creates a new `SCons SexpNode` + new(result) + result.kind = SCons + result.car = car + result.cdr = cdr + +proc newSList*(): SexpNode {.procvar.} = + ## Creates a new `SList SexpNode` + new(result) + result.kind = SList + result.elems = @[] + +proc newSSymbol*(s: string): SexpNode {.procvar.} = + new(result) + result.kind = SSymbol + result.symbol = s + +proc newSSymbolMove(s: string): SexpNode = + new(result) + result.kind = SSymbol + shallowCopy(result.symbol, s) + +proc getStr*(n: SexpNode, default: string = ""): string = + ## Retrieves the string value of a `SString SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SString``. + if n.kind != SString: return default + else: return n.str + +proc getNum*(n: SexpNode, default: BiggestInt = 0): BiggestInt = + ## Retrieves the int value of a `SInt SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SInt``. + if n.kind != SInt: return default + else: return n.num + +proc getFNum*(n: SexpNode, default: float = 0.0): float = + ## Retrieves the float value of a `SFloat SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SFloat``. + if n.kind != SFloat: return default + else: return n.fnum + +proc getSymbol*(n: SexpNode, default: string = ""): string = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind != SSymbol: return default + else: return n.symbol + +proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SNil: return @[] + elif n.kind != SList: return default + else: return n.elems + +proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons = + ## Retrieves the cons value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SCons: return (n.car, n.cdr) + elif n.kind == SList: return (n.elems[0], n.elems[1]) + else: return defaults + +proc sexp*(s: string): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc sexp*(n: BiggestInt): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc sexp*(n: float): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc sexp*(b: bool): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SSymbol + ## SexpNode` with value t or `SNil SexpNode`. + new(result) + if b: + result.kind = SSymbol + result.symbol = "t" + else: + result.kind = SNil + +proc sexp*(elements: openArray[SexpNode]): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SList SexpNode` + new(result) + result.kind = SList + newSeq(result.elems, elements.len) + for i, p in pairs(elements): result.elems[i] = p + +proc sexp*(s: SexpNode): SexpNode = + result = s + +proc toSexp(x: NimNode): NimNode {.compiletime.} = + case x.kind + of nnkBracket: + result = newNimNode(nnkBracket) + for i in 0 .. <x.len: + result.add(toSexp(x[i])) + + else: + result = x + + result = prefix(result, "sexp") + +macro convertSexp*(x: untyped): untyped = + ## Convert an expression to a SexpNode directly, without having to specify + ## `%` for every element. + result = toSexp(x) + +proc `==`* (a,b: SexpNode): bool = + ## Check two nodes for equality + if a.isNil: + if b.isNil: return true + return false + elif b.isNil or a.kind != b.kind: + return false + else: + return case a.kind + of SString: + a.str == b.str + of SInt: + a.num == b.num + of SFloat: + a.fnum == b.fnum + of SNil: + true + of SList: + a.elems == b.elems + of SSymbol: + a.symbol == b.symbol + of SCons: + a.car == b.car and a.cdr == b.cdr + +proc hash* (n:SexpNode): Hash = + ## Compute the hash for a SEXP node + case n.kind + of SList: + result = hash(n.elems) + of SInt: + result = hash(n.num) + of SFloat: + result = hash(n.fnum) + of SString: + result = hash(n.str) + of SNil: + result = hash(0) + of SSymbol: + result = hash(n.symbol) + of SCons: + result = hash(n.car) !& hash(n.cdr) + +proc len*(n: SexpNode): int = + ## If `n` is a `SList`, it returns the number of elements. + ## If `n` is a `JObject`, it returns the number of pairs. + ## Else it returns 0. + case n.kind + of SList: result = n.elems.len + else: discard + +proc `[]`*(node: SexpNode, index: int): SexpNode = + ## Gets the node at `index` in a List. Result is undefined if `index` + ## is out of bounds + assert(not isNil(node)) + assert(node.kind == SList) + return node.elems[index] + +proc add*(father, child: SexpNode) = + ## Adds `child` to a SList node `father`. + assert father.kind == SList + father.elems.add(child) + +# ------------- pretty printing ---------------------------------------------- + +proc indent(s: var string, i: int) = + s.add(spaces(i)) + +proc newIndent(curr, indent: int, ml: bool): int = + if ml: return curr + indent + else: return indent + +proc nl(s: var string, ml: bool) = + if ml: s.add("\n") + +proc escapeJson*(s: string): string = + ## Converts a string `s` to its JSON representation. + result = newStringOfCap(s.len + s.len shr 3) + result.add("\"") + for x in runes(s): + var r = int(x) + if r >= 32 and r <= 127: + var c = chr(r) + case c + of '"': result.add("\\\"") #" # gotta fix nim-mode + of '\\': result.add("\\\\") + else: result.add(c) + else: + result.add("\\u") + result.add(toHex(r, 4)) + result.add("\"") + +proc copy*(p: SexpNode): SexpNode = + ## Performs a deep copy of `a`. + case p.kind + of SString: + result = newSString(p.str) + of SInt: + result = newSInt(p.num) + of SFloat: + result = newSFloat(p.fnum) + of SNil: + result = newSNil() + of SSymbol: + result = newSSymbol(p.symbol) + of SList: + result = newSList() + for i in items(p.elems): + result.elems.add(copy(i)) + of SCons: + result = newSCons(copy(p.car), copy(p.cdr)) + +proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, + lstArr = false, currIndent = 0) = + case node.kind + of SString: + if lstArr: result.indent(currIndent) + result.add(escapeJson(node.str)) + of SInt: + if lstArr: result.indent(currIndent) + result.add($node.num) + of SFloat: + if lstArr: result.indent(currIndent) + result.add($node.fnum) + of SNil: + if lstArr: result.indent(currIndent) + result.add("nil") + of SSymbol: + if lstArr: result.indent(currIndent) + result.add($node.symbol) + of SList: + if lstArr: result.indent(currIndent) + if len(node.elems) != 0: + result.add("(") + result.nl(ml) + for i in 0..len(node.elems)-1: + if i > 0: + result.add(" ") + result.nl(ml) # New Line + toPretty(result, node.elems[i], indent, ml, + true, newIndent(currIndent, indent, ml)) + result.nl(ml) + result.indent(currIndent) + result.add(")") + else: result.add("nil") + of SCons: + if lstArr: result.indent(currIndent) + result.add("(") + toPretty(result, node.car, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(" . ") + toPretty(result, node.cdr, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(")") + +proc pretty*(node: SexpNode, indent = 2): string = + ## Converts `node` to its Sexp Representation, with indentation and + ## on multiple lines. + result = "" + toPretty(result, node, indent) + +proc `$`*(node: SexpNode): string = + ## Converts `node` to its SEXP Representation on one line. + result = "" + toPretty(result, node, 0, false) + +iterator items*(node: SexpNode): SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. + assert node.kind == SList + for i in items(node.elems): + yield i + +iterator mitems*(node: var SexpNode): var SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. Items can be + ## modified. + assert node.kind == SList + for i in mitems(node.elems): + yield i + +proc eat(p: var SexpParser, tok: TTokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) + +proc parseSexp(p: var SexpParser): SexpNode = + ## Parses SEXP from a SEXP Parser `p`. + case p.tok + of tkString: + # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: + result = newSStringMove(p.a) + p.a = "" + discard getTok(p) + of tkInt: + result = newSInt(parseBiggestInt(p.a)) + discard getTok(p) + of tkFloat: + result = newSFloat(parseFloat(p.a)) + discard getTok(p) + of tkNil: + result = newSNil() + discard getTok(p) + of tkSymbol: + result = newSSymbolMove(p.a) + p.a = "" + discard getTok(p) + of tkParensLe: + result = newSList() + discard getTok(p) + while p.tok notin {tkParensRi, tkDot}: + result.add(parseSexp(p)) + if p.tok != tkSpace: break + discard getTok(p) + if p.tok == tkDot: + eat(p, tkDot) + eat(p, tkSpace) + result.add(parseSexp(p)) + result = newSCons(result[0], result[1]) + eat(p, tkParensRi) + of tkSpace, tkDot, tkError, tkParensRi, tkEof: + raiseParseErr(p, "(") + +proc open*(my: var SexpParser, input: Stream) = + ## initializes the parser with an input stream. + lexbase.open(my, input) + my.kind = sexpError + my.a = "" + +proc parseSexp*(s: Stream): SexpNode = + ## Parses from a buffer `s` into a `SexpNode`. + var p: SexpParser + p.open(s) + discard getTok(p) # read first token + result = p.parseSexp() + p.close() + +proc parseSexp*(buffer: string): SexpNode = + ## Parses Sexp from `buffer`. + result = parseSexp(newStringStream(buffer)) + +when isMainModule: + let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") + assert(testSexp[0].getNum == 1) + assert(testSexp[1][0].getNum == 98) + assert(testSexp[2].getElems == @[]) + assert(testSexp[4].getSymbol == "foobar") + assert(testSexp[5].getStr == "foo") + + let alist = parseSexp("""((1 . 2) (2 . "foo"))""") + assert(alist[0].getCons.car.getNum == 1) + assert(alist[0].getCons.cdr.getNum == 2) + assert(alist[1].getCons.cdr.getStr == "foo") + + # Generator: + var j = convertSexp([true, false, "foobar", [1, 2, "baz"]]) + assert($j == """(t nil "foobar" (1 2 "baz"))""") diff --git a/tools/nimsuggest/tester.nim b/tools/nimsuggest/tester.nim new file mode 100644 index 000000000..c90afe3db --- /dev/null +++ b/tools/nimsuggest/tester.nim @@ -0,0 +1,182 @@ +# Tester for nimsuggest. +# Every test file can have a #[!]# comment that is deleted from the input +# before 'nimsuggest' is invoked to ensure this token doesn't make a +# crucial difference for Nim's parser. + +import os, osproc, strutils, streams, re + +type + Test = object + cmd, dest: string + startup: seq[string] + script: seq[(string, string)] + +const + curDir = when defined(windows): "" else: "" + DummyEof = "!EOF!" + +template tpath(): untyped = getAppDir() / "tests" + +proc parseTest(filename: string): Test = + const cursorMarker = "#[!]#" + let nimsug = curDir & addFileExt("nimsuggest", ExeExt) + result.dest = getTempDir() / extractFilename(filename) + result.cmd = nimsug & " --tester " & result.dest + result.script = @[] + result.startup = @[] + var tmp = open(result.dest, fmWrite) + var specSection = 0 + var markers = newSeq[string]() + var i = 1 + for x in lines(filename): + let marker = x.find(cursorMarker)+1 + if marker > 0: + markers.add "\"" & filename & "\";\"" & result.dest & "\":" & $i & ":" & $marker + tmp.writeLine x.replace(cursorMarker, "") + else: + tmp.writeLine x + if x.contains("""""""""): + inc specSection + elif specSection == 1: + if x.startsWith("$nimsuggest"): + result.cmd = x % ["nimsuggest", nimsug, "file", filename] + elif x.startsWith("!"): + if result.cmd.len == 0: + result.startup.add x + else: + result.script.add((x, "")) + elif x.startsWith(">"): + # since 'markers' here are not complete yet, we do the $substitutions + # afterwards + result.script.add((x.substr(1).replaceWord("$path", tpath()), "")) + elif x.len > 0: + # expected output line: + let x = x % ["file", filename] + result.script[^1][1].add x.replace(";;", "\t") & '\L' + # else: ignore empty lines for better readability of the specs + inc i + tmp.close() + # now that we know the markers, substitute them: + for a in mitems(result.script): + a[0] = a[0] % markers + +proc parseCmd(c: string): seq[string] = + # we don't support double quotes for now so that + # we can later support them properly with escapes and stuff. + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while c[i] in {' ', '\t', '\l', '\r'}: inc(i) + case c[i] + of '"': raise newException(ValueError, "double quotes not yet supported: " & c) + of '\'': + var delim = c[i] + inc(i) # skip ' or " + while c[i] != '\0' and c[i] != delim: + add a, c[i] + inc(i) + if c[i] != '\0': inc(i) + of '\0': break + else: + while c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, a) + +proc edit(tmpfile: string; x: seq[string]) = + if x.len != 3 and x.len != 4: + quit "!edit takes two or three arguments" + let f = if x.len >= 4: tpath() / x[3] else: tmpfile + try: + let content = readFile(f) + let newcontent = content.replace(x[1], x[2]) + if content == newcontent: + quit "wrong test case: edit had no effect" + writeFile(f, newcontent) + except IOError: + quit "cannot edit file " & tmpfile + +proc exec(x: seq[string]) = + if x.len != 2: quit "!exec takes one argument" + if execShellCmd(x[1]) != 0: + quit "External program failed " & x[1] + +proc copy(x: seq[string]) = + if x.len != 3: quit "!copy takes two arguments" + let rel = tpath() + copyFile(rel / x[1], rel / x[2]) + +proc del(x: seq[string]) = + if x.len != 2: quit "!del takes one argument" + removeFile(tpath() / x[1]) + +proc runCmd(cmd, dest: string): bool = + result = cmd[0] == '!' + if not result: return + let x = cmd.parseCmd() + case x[0] + of "!edit": + edit(dest, x) + of "!exec": + exec(x) + of "!copy": + copy(x) + of "!del": + del(x) + else: + quit "unkown command: " & cmd + +proc smartCompare(pattern, x: string): bool = + if pattern.contains('*'): + result = match(x, re(escapeRe(pattern).replace("\\x2A","(.*)"), {})) + +proc runTest(filename: string): int = + let s = parseTest filename + for cmd in s.startup: + if not runCmd(cmd, s.dest): + quit "invalid command: " & cmd + let cl = parseCmdLine(s.cmd) + var p = startProcess(command=cl[0], args=cl[1 .. ^1], + options={poStdErrToStdOut, poUsePath, + poInteractive, poDemon}) + let outp = p.outputStream + let inp = p.inputStream + var report = "" + var a = newStringOfCap(120) + try: + # read and ignore anything nimsuggest says at startup: + while outp.readLine(a): + if a == DummyEof: break + for req, resp in items(s.script): + if not runCmd(req, s.dest): + inp.writeLine(req) + inp.flush() + var answer = "" + while outp.readLine(a): + if a == DummyEof: break + answer.add a + answer.add '\L' + if resp != answer and not smartCompare(resp, answer): + report.add "\nTest failed: " & filename + report.add "\n Expected: " & resp + report.add "\n But got: " & answer + finally: + inp.writeLine("quit") + inp.flush() + close(p) + if report.len > 0: + echo report + result = report.len + +proc main() = + var failures = 0 + for x in walkFiles(getAppDir() / "tests/t*.nim"): + echo "Test ", x + failures += runTest(expandFilename(x)) + if failures > 0: + quit 1 + +main() diff --git a/tools/nimsuggest/tests/dep_v1.nim b/tools/nimsuggest/tests/dep_v1.nim new file mode 100644 index 000000000..eae230e85 --- /dev/null +++ b/tools/nimsuggest/tests/dep_v1.nim @@ -0,0 +1,8 @@ + + + + + +type + Foo* = object + x*, y*: int diff --git a/tools/nimsuggest/tests/dep_v2.nim b/tools/nimsuggest/tests/dep_v2.nim new file mode 100644 index 000000000..ab39721c4 --- /dev/null +++ b/tools/nimsuggest/tests/dep_v2.nim @@ -0,0 +1,9 @@ + + + + + +type + Foo* = object + x*, y*: int + z*: string diff --git a/tools/nimsuggest/tests/tdef1.nim b/tools/nimsuggest/tests/tdef1.nim new file mode 100644 index 000000000..960ffad1c --- /dev/null +++ b/tools/nimsuggest/tests/tdef1.nim @@ -0,0 +1,16 @@ +discard """ +$nimsuggest --tester $file +>def $1 +def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +>def $1 +def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +""" + +proc hello() string = + ## Return hello + "Hello" + +hel#[!]#lo() + +# v uncompleted id for sug (13,2) +he diff --git a/tools/nimsuggest/tests/tdot1.nim b/tools/nimsuggest/tests/tdot1.nim new file mode 100644 index 000000000..bcd44cd84 --- /dev/null +++ b/tools/nimsuggest/tests/tdot1.nim @@ -0,0 +1,14 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;$file;;11;;4;;"";;100 +sug;;skField;;y;;int;;$file;;11;;7;;"";;100 +sug;;skProc;;tdot1.main;;proc (f: Foo);;$file;;13;;5;;"";;100 +""" + +type + Foo = object + x, y: int + +proc main(f: Foo) = + f.#[!]# diff --git a/tools/nimsuggest/tests/tdot2.nim b/tools/nimsuggest/tests/tdot2.nim new file mode 100644 index 000000000..a58ac818b --- /dev/null +++ b/tools/nimsuggest/tests/tdot2.nim @@ -0,0 +1,29 @@ +# Test basic editing. We replace the 'false' by 'true' to +# see whether then the z field is suggested. + +const zField = 0i32 + +type + Foo = object + x, y: int + when zField == 1i32: + z: string + +proc main(f: Foo) = + f.#[!]# + +# the tester supports the spec section at the bottom of the file and +# this way, the line numbers more often stay the same +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;$file;;8;;4;;"";;100 +sug;;skField;;y;;int;;$file;;8;;7;;"";;100 +sug;;skProc;;tdot2.main;;proc (f: Foo);;$file;;12;;5;;"";;100 +!edit 0i32 1i32 +>sug $1 +sug;;skField;;x;;int;;$file;;8;;4;;"";;100 +sug;;skField;;y;;int;;$file;;8;;7;;"";;100 +sug;;skField;;z;;string;;$file;;10;;6;;"";;100 +sug;;skProc;;tdot2.main;;proc (f: Foo);;$file;;12;;5;;"";;100 +""" diff --git a/tools/nimsuggest/tests/tdot3.nim b/tools/nimsuggest/tests/tdot3.nim new file mode 100644 index 000000000..5badde867 --- /dev/null +++ b/tools/nimsuggest/tests/tdot3.nim @@ -0,0 +1,27 @@ +# Test basic module dependency recompilations. + +import dep + +proc main(f: Foo) = + f.#[!]# + +# the tester supports the spec section at the bottom of the file and +# this way, the line numbers more often stay the same + +discard """ +!copy dep_v1.nim dep.nim +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;*dep.nim;;8;;4;;"";;100 +sug;;skField;;y;;int;;*dep.nim;;8;;8;;"";;100 +sug;;skProc;;tdot3.main;;proc (f: Foo);;$file;;5;;5;;"";;100 + +!copy dep_v2.nim dep.nim +>mod $path/dep.nim +>sug $1 +sug;;skField;;x;;int;;*dep.nim;;8;;4;;"";;100 +sug;;skField;;y;;int;;*dep.nim;;8;;8;;"";;100 +sug;;skField;;z;;string;;*dep.nim;;9;;4;;"";;100 +sug;;skProc;;tdot3.main;;proc (f: Foo);;$file;;5;;5;;"";;100 +!del dep.nim +""" diff --git a/tools/nimsuggest/tests/tinclude.nim b/tools/nimsuggest/tests/tinclude.nim new file mode 100644 index 000000000..77492d745 --- /dev/null +++ b/tools/nimsuggest/tests/tinclude.nim @@ -0,0 +1,7 @@ +discard """ +$nimsuggest --tester compiler/nim.nim +>def compiler/semexprs.nim:13:50 +def;;skType;;ast.PSym;;PSym;;*ast.nim;;668;;2;;"";;100 +>def compiler/semexprs.nim:13:50 +def;;skType;;ast.PSym;;PSym;;*ast.nim;;668;;2;;"";;100 +""" diff --git a/tools/nimsuggest/tests/tstrutils.nim b/tools/nimsuggest/tests/tstrutils.nim new file mode 100644 index 000000000..f5cda9505 --- /dev/null +++ b/tools/nimsuggest/tests/tstrutils.nim @@ -0,0 +1,9 @@ +discard """ +$nimsuggest --tester lib/pure/strutils.nim +>def lib/pure/strutils.nim:2300:6 +def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"";;100 +""" + +# Line 2300 in strutils.nim is doAssert and this is unlikely to change +# soon since there are a whole lot of doAsserts there. + diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 65af67216..9a21a0fb5 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -29,7 +29,7 @@ type TRssItem = object year, month, day, title, url, content: string TAction = enum - actAll, actOnlyWebsite, actPdf, actJson2 + actAll, actOnlyWebsite, actPdf, actJson2, actOnlyDocs Sponsor = object logo: string @@ -72,7 +72,7 @@ include "website.tmpl" # ------------------------- configuration file ------------------------------- const - version = "0.7" + version = "0.8" usage = "nimweb - Nim Website Generator Version " & version & """ (c) 2015 Andreas Rumpf @@ -85,11 +85,13 @@ Options: -v, --version shows the version --website only build the website, not the full documentation --pdf build the PDF version of the documentation + --json2 build JSON of the documentation + --onlyDocs build only the documentation Compile_options: will be passed to the Nim compiler """ - rYearMonthDay = r"(\d{4})_(\d{2})_(\d{2})" + rYearMonthDay = r"on\s+(\d{2})\/(\d{2})\/(\d{4})" rssUrl = "http://nim-lang.org/news.xml" rssNewsUrl = "http://nim-lang.org/news.html" activeSponsors = "web/sponsors.csv" @@ -157,6 +159,7 @@ proc parseCmdLine(c: var TConfigData) = of "website": action = actOnlyWebsite of "pdf": action = actPdf of "json2": action = actJson2 + of "onlydocs": action = actOnlyDocs of "googleanalytics": c.gaId = val c.nimArgs.add("--doc.googleAnalytics:" & val & " ") @@ -315,6 +318,7 @@ proc buildDoc(c: var TConfigData, destPath: string) = exec(findNim() & " buildIndex -o:$1/theindex.html $1" % [destPath]) proc buildPdfDoc(c: var TConfigData, destPath: string) = + createDir(destPath) if os.execShellCmd("pdflatex -version") != 0: echo "pdflatex not found; no PDF documentation generated" else: @@ -347,6 +351,7 @@ proc buildAddDoc(c: var TConfigData, destPath: string) = proc parseNewsTitles(inputFilename: string): seq[TRssItem] = # Goes through each news file, returns its date/title. result = @[] + var matches: array[3, string] let reYearMonthDay = re(rYearMonthDay) for kind, path in walkDir(inputFilename): let (dir, name, ext) = path.splitFile @@ -354,8 +359,8 @@ proc parseNewsTitles(inputFilename: string): seq[TRssItem] = let content = readFile(path) let title = content.splitLines()[0] let urlPath = "news/" & name & ".html" - if name =~ reYearMonthDay: - result.add(TRssItem(year: matches[0], month: matches[1], day: matches[2], + if content.find(reYearMonthDay, matches) >= 0: + result.add(TRssItem(year: matches[2], month: matches[1], day: matches[0], title: title, url: "http://nim-lang.org/" & urlPath, content: content)) result.reverse() @@ -501,11 +506,19 @@ proc buildWebsite(c: var TConfigData) = proc main(c: var TConfigData) = buildWebsite(c) buildJS("web/upload") - buildAddDoc(c, "web/upload") - buildDocSamples(c, "web/upload") - buildDoc(c, "web/upload") - buildDocSamples(c, "doc") - buildDoc(c, "doc") + const docup = "web/upload/" & NimVersion + createDir(docup) + buildAddDoc(c, docup) + buildDocSamples(c, docup) + buildDoc(c, docup) + createDir("doc/html") + buildDocSamples(c, "doc/html") + buildDoc(c, "doc/html") + +proc onlyDocs(c: var TConfigData) = + createDir("doc/html") + buildDocSamples(c, "doc/html") + buildDoc(c, "doc/html") proc json2(c: var TConfigData) = const destPath = "web/json2" @@ -526,6 +539,7 @@ parseCmdLine(c) parseIniFile(c) case action of actOnlyWebsite: buildWebsite(c) -of actPdf: buildPdfDoc(c, "doc") +of actPdf: buildPdfDoc(c, "doc/pdf") +of actOnlyDocs: onlyDocs(c) of actAll: main(c) of actJson2: json2(c) diff --git a/tools/noprefix.nim b/tools/noprefix.nim deleted file mode 100644 index d16c2b520..000000000 --- a/tools/noprefix.nim +++ /dev/null @@ -1,32 +0,0 @@ -# strip those silly GTK/ATK prefixes... - -import - expandimportc, os - -const - filelist = [ - ("sdl/sdl", "sdl"), - ("sdl/sdl_net", "sdl"), - ("sdl/sdl_gfx", "sdl"), - ("sdl/sdl_image", "sdl"), - ("sdl/sdl_mixer_nosmpeg", "sdl"), - ("sdl/sdl_mixer", "sdl"), - ("sdl/sdl_ttf", "sdl"), - ("sdl/smpeg", "sdl"), - - ("libcurl", "curl"), - ("mysql", "mysql"), - ("postgres", ""), - ("sqlite3", "sqlite3"), - - ("pcre/pcre", "pcre") - ] - -proc createDirs = - createDir("lib/newwrap/sdl") - createDir("lib/newwrap/pcre") - -for filename, prefix in items(filelist): - var f = addFileExt(filename, "nim") - main("lib/wrappers" / f, "lib/newwrap" / f, prefix) - diff --git a/start.bat b/tools/start.bat index 685932c47..a4475fac7 100644 --- a/start.bat +++ b/tools/start.bat @@ -1,6 +1,8 @@ @echo off REM COLOR 0A -SET NIMPATH=%~dp0 +SET NIMPATH=%~dp0\.. SET PATH=%NIMPATH%\bin;%NIMPATH%\dist\mingw\bin;%PATH% -cmd +cd %NIMPATH% +cmd + diff --git a/tools/vccenv/vccenv.nim b/tools/vccenv/vccenv.nim new file mode 100644 index 000000000..a335efd10 --- /dev/null +++ b/tools/vccenv/vccenv.nim @@ -0,0 +1,58 @@ +import strtabs, os, osproc, streams, strutils + +const + comSpecEnvKey = "ComSpec" + vsComnToolsEnvKeys = [ + "VS140COMNTOOLS", + "VS130COMNTOOLS", + "VS120COMNTOOLS", + "VS110COMNTOOLS", + "VS100COMNTOOLS", + "VS90COMNTOOLS" + ] + vcvarsallRelativePath = joinPath("..", "..", "VC", "vcvarsall") + +proc getVsComnToolsPath*(): TaintedString = + for vsComnToolsEnvKey in vsComnToolsEnvKeys: + let vsComnToolsEnvVal = getEnv vsComnToolsEnvKey + if vsComnToolsEnvVal.len > 0: + return vsComnToolsEnvVal + +proc getVccEnv*(platform: string, windowsStoreSdk: bool = false, + sdkVersion: string = nil): StringTableRef = + var comSpecCommandString = getEnv comSpecEnvKey + if comSpecCommandString.len == 0: + comSpecCommandString = "cmd" + + let vsComnToolsPath = getVsComnToolsPath() + if vsComnToolsPath.len < 1: + return nil + let vcvarsallPath = expandFilename joinPath(vsComnToolsPath, vcvarsallRelativePath) + + var vcvarsallArgs: seq[string] = @[] + if platform.len > 0: + vcvarsallArgs.add(platform) + if windowsStoreSdk: + vcvarsallArgs.add("store") + if sdkVersion.len > 0: + vcvarsallArgs.add(sdkVersion) + let vcvarsallArgString = vcvarsallArgs.join(" ") + + var vcvarsallCommandString: string + if vcvarsallArgString.len > 0: + vcvarsallCommandString = "\"$1\" $2" % [vcvarsallPath, vcvarsallArgString] + else: + vcvarsallCommandString = vcvarsallPath + + let vcvarsallExecCommand = "\"$1\" /C \"$2 && SET\"" % + [comSpecCommandString, vcvarsallCommandString] + when defined(release): + let vccvarsallOptions = {poEvalCommand, poDemon} + else: + let vccvarsallOptions = {poEchoCmd, poEvalCommand, poDemon} + let vcvarsallStdOut = execProcess(vcvarsallExecCommand, options = vccvarsallOptions) + result = newStringTable(modeCaseInsensitive) + for line in vcvarsallStdOut.splitLines: + let idx = line.find('=') + if idx > 0: + result[line[0..(idx - 1)]] = line[(idx + 1)..(line.len - 1)] diff --git a/tools/vccenv/vccexe.nim b/tools/vccenv/vccexe.nim new file mode 100644 index 000000000..892246830 --- /dev/null +++ b/tools/vccenv/vccexe.nim @@ -0,0 +1,66 @@ +import strutils, strtabs, os, osproc, vccenv + +when defined(release): + let vccOptions = {poParentStreams} +else: + let vccOptions = {poEchoCmd, poParentStreams} + +const + platformPrefix = "--platform" + winstorePrefix = "--winstore" + sdkversionPrefix = "--sdkversion" + + platformSepIdx = platformPrefix.len + sdkversionSepIdx = sdkversionPrefix.len + + HelpText = """ ++-----------------------------------------------------------------+ +| Microsoft C/C++ compiler wrapper for Nim | +| (c) 2016 Fredrik Høisæther Rasch | ++-----------------------------------------------------------------+ + +Usage: + vccexe [options] [compileroptions] +Options: + --platform:<arch> Specify the Compiler Platform Tools architecture + <arch>: x86 | amd64 | arm | x86_amd64 | x86_arm | amd64_x86 | amd64_arm + --winstore Use Windows Store (rather than desktop) development tools + --sdkversion:<v> Use a specific Windows SDK version: + <v> is either the full Windows 10 SDK version number or + "8.1" to use the windows 8.1 SDK + +Other command line arguments are passed on to the +Microsoft C/C++ compiler for the specified SDK toolset +""" + +when isMainModule: + var platformArg: string = nil + var sdkVersionArg: string = nil + var storeArg: bool = false + + var clArgs: seq[TaintedString] = @[] + + var wrapperArgs = commandLineParams() + for wargv in wrapperArgs: + # Check whether the current argument contains -- prefix + if wargv.startsWith(platformPrefix): # Check for platform + platformArg = wargv.substr(platformSepIdx + 1) + elif wargv == winstorePrefix: # Check for winstore + storeArg = true + elif wargv.startsWith(sdkversionPrefix): # Check for sdkversion + sdkVersionArg = wargv.substr(sdkversionSepIdx + 1) + else: # Regular cl.exe argument -> store for final cl.exe invocation + if (wargv.len == 2) and (wargv[1] == '?'): + echo HelpText + clArgs.add(wargv) + + var vccEnvStrTab = getVccEnv(platformArg, storeArg, sdkVersionArg) + if vccEnvStrTab != nil: + for vccEnvKey, vccEnvVal in vccEnvStrTab: + putEnv(vccEnvKey, vccEnvVal) + let vccProcess = startProcess( + "cl.exe", + args = clArgs, + options = vccOptions + ) + quit vccProcess.waitForExit() diff --git a/tools/website.tmpl b/tools/website.tmpl index 2801fea96..9aa64310d 100644 --- a/tools/website.tmpl +++ b/tools/website.tmpl @@ -61,13 +61,13 @@ <div id="slideshow"> <!-- slides --> <div id="slide0" class="active niaslide"> - <a href="sponsors.html"> - <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/> + <a href="news/e030_nim_in_action_in_production.html"> + <img src="${rootDir}assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!"/> </a> </div> <div id="slide1" class="niaslide"> - <a href="news/2016_01_27_nim_in_action_is_now_available.html"> - <img src="${rootDir}assets/niminaction/banner.jpg" alt="New in Manning Early Access Program: Nim in Action!"/> + <a href="sponsors.html"> + <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/> </a> </div> <div id="slide2" class="codeslide2"> diff --git a/web/assets/niminaction/banner2.png b/web/assets/niminaction/banner2.png new file mode 100644 index 000000000..3cabd195d --- /dev/null +++ b/web/assets/niminaction/banner2.png Binary files differdiff --git a/web/download.rst b/web/download.rst index cf0841577..c388dd132 100644 --- a/web/download.rst +++ b/web/download.rst @@ -3,17 +3,34 @@ Download the compiler You can download the latest version of the Nim compiler here. -Binaries --------- +Windows +------- -Right now binaries are only provided for Windows. You can download -an installer for both 32 bit and 64 bit versions of Windows below. These +Zips +%%%% + +We now encourage you to install via the provided zipfiles: + +* | 32 bit: `nim-0.15.2_x32.zip <download/nim-0.15.2_x32.zip>`_ + | SHA-256 0f1bfb74751f55e090140a361c08e9f39f1dd03f1f0c070c061f2d5049ab9f96 +* | 64 bit: `nim-0.15.2_x64.zip <download/nim-0.15.2_x64.zip>`_ + | SHA-256 ceea42de6ebcd41032ee51f04526dc4cf2cbb0958ca6ad2321cf21944e05f553 + +Unzip these where you want and optionally run ``finish.exe`` to +detect your MingW environment. + +Exes +%%%% + +You can download an installer for both 32 bit and 64 bit versions of +Windows below. Note that these installers have some known issues and +so will unlikely to be provided further in the future. These installers have everything you need to use Nim, including a C compiler. -* | 32 bit: `nim-0.15.0_x32.exe <download/nim-0.15.0_x32.exe>`_ - | SHA-256 0ca8931e3369735bbafdf93de98a8fd0f425870f1173845e7601922a5e00c3c2 -* | 64 bit: `nim-0.15.0_x64.exe <download/nim-0.15.0_x64.exe>`_ - | SHA-256 7bb9321cd9fb2860d36ee9b248e0202d7d4e36e2272a2f128fbce96fd4a9bfd6 +* | 32 bit: `nim-0.15.2_x32.exe <download/nim-0.15.2_x32.exe>`_ + | SHA-256 8d648295dbd59cb315c98926a1da9f1f68773a1a2ef3d9d4c91c59387167efa3 +* | 64 bit: `nim-0.15.2_x64.exe <download/nim-0.15.2_x64.exe>`_ + | SHA-256 8c7efc6571921c2d2e5e995f801d4229ea1de19fbdabdcba1628307bd4612392 These installers also include Aporia, Nimble and other useful Nim tools to get you started with Nim development! @@ -31,8 +48,8 @@ like systems. Firstly, download this archive: -* | `nim-0.15.0.tar.xz (4.5MB) <download/nim-0.15.0.tar.xz>`_ - | SHA-256 c514535050b2b2156147bbe6e23aafe07cd996b2afa2c81fa9a09e1cd8c669fb +* | `nim-0.15.2.tar.xz (4.5MB) <download/nim-0.15.2.tar.xz>`_ + | SHA-256 905df2316262aa2cbacae067acf45fc05c2a71c8c6fde1f2a70c927ebafcfe8a Extract the archive. Then copy the extracted files into your chosen installation directory, ideally somewhere in your home directory. @@ -44,7 +61,7 @@ Now open a terminal and follow these instructions: ``cd ~/programs/nim``. * run ``sh build.sh``. * Add ``$your_install_dir/bin`` to your PATH. -* To build associated tools like ``nimble`` and ``nimsuggest`` run ``nim e install_tools.nims``. +* To build associated tools like ``nimble`` and ``nimsuggest`` run ``nim c koch && koch tools``. After restarting your terminal, you should be able to run ``nim -v`` which should show you the version of Nim you just installed. diff --git a/web/news.rst b/web/news.rst index f819c384c..e8d97b69e 100644 --- a/web/news.rst +++ b/web/news.rst @@ -2,102 +2,108 @@ News ==== -`2016-09-30 Nim Version 0.15.0 released <news/2016_09_00_version_0_15_0_released.html>`_ +`2016-11-20 Nim in Action is going into production! <news/e030_nim_in_action_in_production.html>`_ =================================== -`2016-09-03 Nim Community Survey results <news/2016_09_03_nim_community_survey_results.html>`_ +`2016-10-23 Nim Version 0.15.2 released <news/e028_version_0_15_2.html>`_ =================================== -`2016-08-06 BountySource Update: The Road to v1.0 <news/2016_08_06_bountysource_update_the_road_to_v10.html>`_ +`2016-09-30 Nim Version 0.15.0 released <news/e027_version_0_15_0.html>`_ =================================== -`2016-06-23 Launching the 2016 Nim community survey <news/2016_06_23_launching_the_2016_nim_community_survey.html>`_ +`2016-09-03 Nim Community Survey results <news/e026_survey_results.html>`_ =================================== -`2016-06-11 Version 0.14.2 released <news/2016_06_11_version_0_14_2_released.html>`_ +`2016-08-06 BountySource Update: The Road to v1.0 <news/e025_bountysource_update.html>`_ =================================== -`2016-06-07 Version 0.14.0 released <news/2016_06_07_version_0_14_0_released.html>`_ +`2016-06-23 Launching the 2016 Nim community survey <news/e024_survey.html>`_ =================================== -`2016-06-04 Meet our BountySource sponsors <news/2016_06_04_meet_our_bountysource_sponsors.html>`_ +`2016-06-11 Version 0.14.2 released <news/e023_version_0_14_2.html>`_ =================================== -`2016-01-27 Nim in Action is now available! <news/2016_01_27_nim_in_action_is_now_available.html>`_ +`2016-06-07 Version 0.14.0 released <news/e022_version_0_14_0.html>`_ +=================================== + +`2016-06-04 Meet our BountySource sponsors <news/e021_meet_sponsors.html>`_ +=================================== + +`2016-01-27 Nim in Action is now available! <news/e020_nim_in_action.html>`_ ================================== -`2016-01-18 Version 0.13.0 released <news/2016_01_18_version_0_13_0_released.html>`_ +`2016-01-18 Version 0.13.0 released <news/e019_version_0_13_0.html>`_ ================================== -`2016-01-18 Andreas Rumpf's talk at OSCON Amsterdam <news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.html>`_ +`2016-01-18 Andreas Rumpf's talk at OSCON Amsterdam <news/e018_oscon_amsterdam.html>`_ ================================================== -`2015-10-27 Version 0.12.0 released <news/2015_10_27_version_0_12_0_released.html>`_ +`2015-10-27 Version 0.12.0 released <news/e017_version_0_12_0.html>`_ ================================== -`2015-10-16 First Nim conference <news/2015_10_16_first_nim_conference.html>`_ +`2015-10-16 First Nim conference <news/e016_nim_conf1.html>`_ =============================== -`2015-05-04 Version 0.11.2 released <news/2015_05_04_version_0_11_2_released.html>`_ +`2015-05-04 Version 0.11.2 released <news/e015_version_0_11_2.html>`_ ================================== -`2015-04-30 Version 0.11.0 released <news/2015_04_30_version_0_11_0_released.html>`_ +`2015-04-30 Version 0.11.0 released <news/e014_version_0_11_0.html>`_ ================================== -`2014-12-29 Version 0.10.2 released <news/2014_12_29_version_0_10_2_released.html>`_ +`2014-12-29 Version 0.10.2 released <news/e013_version_0_10_2.html>`_ ================================== -`2014-10-19 Version 0.9.6 released <news/2014_10_19_version_0_9_6_released.html>`_ +`2014-10-19 Version 0.9.6 released <news/e012_version_0_9_6.html>`_ ================================= -`2014-04-21 Version 0.9.4 released <news/2014_04_21_version_0_9_4_released.html>`_ +`2014-04-21 Version 0.9.4 released <news/e011_version_0_9_4.html>`_ ================================= -`2014-02-11 Nimrod Featured in Dr. Dobb's Journal <news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.html>`_ +`2014-02-11 Nimrod Featured in Dr. Dobb's Journal <news/e010_dr_dobbs_journal.html>`_ ================================================ -`2014-01-15 Andreas Rumpf's talk on Nimrod at Strange Loop 2013 is now online <news/2014_01_15_andreas_rumpfs_talk_on_nimrod.html>`_ +`2014-01-15 Andreas Rumpf's talk on Nimrod at Strange Loop 2013 is now online <news/e009_andreas_rumpfs_talk.html>`_ ============================================================================ -`2013-05-20 New website design! <news/2013_05_20_new_website_design.html>`_ +`2013-05-20 New website design! <news/e008_new_website.html>`_ ============================== -`2013-05-20 Version 0.9.2 released <news/2013_05_20_version_0_9_2_released.html>`_ +`2013-05-20 Version 0.9.2 released <news/e007_version_0_9_2.html>`_ ================================= -`2012-09-23 Version 0.9.0 released <news/2012_09_23_version_0_9_0_released.html>`_ +`2012-09-23 Version 0.9.0 released <news/e006_version_0_9_0.html>`_ ================================= -`2012-02-09 Version 0.8.14 released <news/2012_02_09_version_0_8_14_released.html>`_ +`2012-02-09 Version 0.8.14 released <news/e005_version_0_8_14.html>`_ ================================== -`2011-07-10 Version 0.8.12 released <news/2011_07_10_version_0_8_12_released.html>`_ +`2011-07-10 Version 0.8.12 released <news/e004_version_0_8_12.html>`_ ================================== -`2010-10-20 Version 0.8.10 released <news/2010_10_20_version_0_8_10_released.html>`_ +`2010-10-20 Version 0.8.10 released <news/e003_version_0_8_10.html>`_ ================================== -`2010-03-14 Version 0.8.8 released <news/2010_03_14_version_0_8_8_released.html>`_ +`2010-03-14 Version 0.8.8 released <news/e002_version_0_8_8.html>`_ ================================= -`2009-12-21 Version 0.8.6 released <news/2009_12_21_version_0_8_6_released.html>`_ +`2009-12-21 Version 0.8.6 released <news/e001_version_0_8_6.html>`_ ================================= diff --git a/web/news/2009_12_21_version_0_8_6_released.rst b/web/news/e001_version_0_8_6.rst index 019168a44..019168a44 100644 --- a/web/news/2009_12_21_version_0_8_6_released.rst +++ b/web/news/e001_version_0_8_6.rst diff --git a/web/news/2010_03_14_version_0_8_8_released.rst b/web/news/e002_version_0_8_8.rst index 2df476814..2df476814 100644 --- a/web/news/2010_03_14_version_0_8_8_released.rst +++ b/web/news/e002_version_0_8_8.rst diff --git a/web/news/2010_10_20_version_0_8_10_released.rst b/web/news/e003_version_0_8_10.rst index f72c0076c..f72c0076c 100644 --- a/web/news/2010_10_20_version_0_8_10_released.rst +++ b/web/news/e003_version_0_8_10.rst diff --git a/web/news/2011_07_10_version_0_8_12_released.rst b/web/news/e004_version_0_8_12.rst index 5f154b2db..5f154b2db 100644 --- a/web/news/2011_07_10_version_0_8_12_released.rst +++ b/web/news/e004_version_0_8_12.rst diff --git a/web/news/2012_02_09_version_0_8_14_released.rst b/web/news/e005_version_0_8_14.rst index 4050c8b93..4050c8b93 100644 --- a/web/news/2012_02_09_version_0_8_14_released.rst +++ b/web/news/e005_version_0_8_14.rst diff --git a/web/news/2012_09_23_version_0_9_0_released.rst b/web/news/e006_version_0_9_0.rst index 5635ca94c..5635ca94c 100644 --- a/web/news/2012_09_23_version_0_9_0_released.rst +++ b/web/news/e006_version_0_9_0.rst diff --git a/web/news/2013_05_20_version_0_9_2_released.rst b/web/news/e007_version_0_9_2.rst index 89352c06c..89352c06c 100644 --- a/web/news/2013_05_20_version_0_9_2_released.rst +++ b/web/news/e007_version_0_9_2.rst diff --git a/web/news/2013_05_20_new_website_design.rst b/web/news/e008_new_website.rst index b36cc99dd..b36cc99dd 100644 --- a/web/news/2013_05_20_new_website_design.rst +++ b/web/news/e008_new_website.rst diff --git a/web/news/2014_01_15_andreas_rumpfs_talk_on_nimrod.rst b/web/news/e009_andreas_rumpfs_talk.rst index 00cc5e101..00cc5e101 100644 --- a/web/news/2014_01_15_andreas_rumpfs_talk_on_nimrod.rst +++ b/web/news/e009_andreas_rumpfs_talk.rst diff --git a/web/news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.rst b/web/news/e010_dr_dobbs_journal.rst index b48ccf31f..b48ccf31f 100644 --- a/web/news/2014_02_11_nimrod_featured_in_dr_dobbs_journal.rst +++ b/web/news/e010_dr_dobbs_journal.rst diff --git a/web/news/2014_04_21_version_0_9_4_released.rst b/web/news/e011_version_0_9_4.rst index 2714c5c78..2714c5c78 100644 --- a/web/news/2014_04_21_version_0_9_4_released.rst +++ b/web/news/e011_version_0_9_4.rst diff --git a/web/news/2014_10_19_version_0_9_6_released.rst b/web/news/e012_version_0_9_6.rst index 7a148aaa5..7a148aaa5 100644 --- a/web/news/2014_10_19_version_0_9_6_released.rst +++ b/web/news/e012_version_0_9_6.rst diff --git a/web/news/2014_12_29_version_0_10_2_released.rst b/web/news/e013_version_0_10_2.rst index ad8afa3bf..ad8afa3bf 100644 --- a/web/news/2014_12_29_version_0_10_2_released.rst +++ b/web/news/e013_version_0_10_2.rst diff --git a/web/news/2015_04_30_version_0_11_0_released.rst b/web/news/e014_version_0_11_0.rst index a8a58f2ae..a8a58f2ae 100644 --- a/web/news/2015_04_30_version_0_11_0_released.rst +++ b/web/news/e014_version_0_11_0.rst diff --git a/web/news/2015_05_04_version_0_11_2_released.rst b/web/news/e015_version_0_11_2.rst index 273182340..273182340 100644 --- a/web/news/2015_05_04_version_0_11_2_released.rst +++ b/web/news/e015_version_0_11_2.rst diff --git a/web/news/2015_10_16_first_nim_conference.rst b/web/news/e016_nim_conf1.rst index 228bffd28..228bffd28 100644 --- a/web/news/2015_10_16_first_nim_conference.rst +++ b/web/news/e016_nim_conf1.rst diff --git a/web/news/2015_10_27_version_0_12_0_released.rst b/web/news/e017_version_0_12_0.rst index 63088f9e2..63088f9e2 100644 --- a/web/news/2015_10_27_version_0_12_0_released.rst +++ b/web/news/e017_version_0_12_0.rst diff --git a/web/news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.rst b/web/news/e018_oscon_amsterdam.rst index fcb4a8794..fcb4a8794 100644 --- a/web/news/2016_01_18_andreas_rumpfs_talk_at_oscon_amsterdam.rst +++ b/web/news/e018_oscon_amsterdam.rst diff --git a/web/news/2016_01_18_version_0_13_0_released.rst b/web/news/e019_version_0_13_0.rst index 2c8e66fa3..2c8e66fa3 100644 --- a/web/news/2016_01_18_version_0_13_0_released.rst +++ b/web/news/e019_version_0_13_0.rst diff --git a/web/news/2016_01_27_nim_in_action_is_now_available.rst b/web/news/e020_nim_in_action.rst index 33bcb7947..33bcb7947 100644 --- a/web/news/2016_01_27_nim_in_action_is_now_available.rst +++ b/web/news/e020_nim_in_action.rst diff --git a/web/news/2016_06_04_meet_our_bountysource_sponsors.rst b/web/news/e021_meet_sponsors.rst index 0bfb472c5..0bfb472c5 100644 --- a/web/news/2016_06_04_meet_our_bountysource_sponsors.rst +++ b/web/news/e021_meet_sponsors.rst diff --git a/web/news/2016_06_07_version_0_14_0_released.rst b/web/news/e022_version_0_14_0.rst index 6634d0053..6634d0053 100644 --- a/web/news/2016_06_07_version_0_14_0_released.rst +++ b/web/news/e022_version_0_14_0.rst diff --git a/web/news/2016_06_11_version_0_14_2_released.rst b/web/news/e023_version_0_14_2.rst index cbfe52713..cbfe52713 100644 --- a/web/news/2016_06_11_version_0_14_2_released.rst +++ b/web/news/e023_version_0_14_2.rst diff --git a/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst b/web/news/e024_survey.rst index 0b87577aa..0b87577aa 100644 --- a/web/news/2016_06_23_launching_the_2016_nim_community_survey.rst +++ b/web/news/e024_survey.rst diff --git a/web/news/2016_08_06_bountysource_update_the_road_to_v10.rst b/web/news/e025_bountysource_update.rst index 00ca7022e..00ca7022e 100644 --- a/web/news/2016_08_06_bountysource_update_the_road_to_v10.rst +++ b/web/news/e025_bountysource_update.rst diff --git a/web/news/2016_09_03_nim_community_survey_results.rst b/web/news/e026_survey_results.rst index 106dce0e4..106dce0e4 100644 --- a/web/news/2016_09_03_nim_community_survey_results.rst +++ b/web/news/e026_survey_results.rst diff --git a/web/news/2016_09_30_version_0_15_0_released.rst b/web/news/e027_version_0_15_0.rst index 47c02e9e4..47c02e9e4 100644 --- a/web/news/2016_09_30_version_0_15_0_released.rst +++ b/web/news/e027_version_0_15_0.rst diff --git a/web/news/e028_version_0_15_2.rst b/web/news/e028_version_0_15_2.rst new file mode 100644 index 000000000..601a26646 --- /dev/null +++ b/web/news/e028_version_0_15_2.rst @@ -0,0 +1,77 @@ +Version 0.15.2 released +======================= + +.. container:: metadata + + Posted by Andreas Rumpf on 23/10/2016 + +We're happy to announce that the latest release of Nim, version 0.15.2, is now +available! + +As always, you can grab the latest version from the +`downloads page <http://nim-lang.org/download.html>`_. + +This release is a pure bugfix release fixing the most pressing issues and +regressions of 0.15.0. For Windows we now provide zipfiles in addition to the +NSIS based installer which proves to be hard to maintain and after all these +months still has serious issues. So we encourage you download the .zip +file instead of the .exe file! Unzip it somewhere, run ``finish.exe`` to +detect your MingW installation, done. ``finish.exe`` can also set your PATH +environment variable. + + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-09-30+..+2016-10-23%22+>`_. + + +- Fixed "`NimMain` not exported in DLL, but `NimMainInner` is" + (`#4840 <https://github.com/nim-lang/Nim/issues/4840>`_) +- Fixed "Tables clear seems to be broken" + (`#4844 <https://github.com/nim-lang/Nim/issues/4844>`_) +- Fixed "compiler: internal error" + (`#4845 <https://github.com/nim-lang/Nim/issues/4845>`_) +- Fixed "trivial macro breaks type checking in the compiler" + (`#4608 <https://github.com/nim-lang/Nim/issues/4608>`_) +- Fixed "derived generic types with static[T] breaks type checking in v0.15.0 (worked in v0.14.2)" + (`#4863 <https://github.com/nim-lang/Nim/issues/4863>`_) +- Fixed "xmlparser.parseXml is not recognised as GC-safe" + (`#4899 <https://github.com/nim-lang/Nim/issues/4899>`_) +- Fixed "async makes generics instantiate only once" + (`#4856 <https://github.com/nim-lang/Nim/issues/4856>`_) +- Fixed "db_common docs aren't generated" + (`#4895 <https://github.com/nim-lang/Nim/issues/4895>`_) +- Fixed "rdstdin disappeared from documentation index" + (`#3755 <https://github.com/nim-lang/Nim/issues/3755>`_) +- Fixed "ICE on template call resolution" + (`#4875 <https://github.com/nim-lang/Nim/issues/4875>`_) +- Fixed "Invisible code-block" + (`#3078 <https://github.com/nim-lang/Nim/issues/3078>`_) +- Fixed "nim doc does not generate doc comments correctly" + (`#4913 <https://github.com/nim-lang/Nim/issues/4913>`_) +- Fixed "nim doc2 fails on ARM when running against lib/pure/coro.nim" + (`#4879 <https://github.com/nim-lang/Nim/issues/4879>`_) +- Fixed "xmlparser does not unescape correctly" + (`#1518 <https://github.com/nim-lang/Nim/issues/1518>`_) +- Fixed "[docs] mysterious "raise hook"" + (`#3485 <https://github.com/nim-lang/Nim/issues/3485>`_) +- Fixed "assertion failure in non-release Nim when compiling NimYAML" + (`#4869 <https://github.com/nim-lang/Nim/issues/4869>`_) +- Fixed "A closure causes nimscript to fail with unhandled exception" + (`#4906 <https://github.com/nim-lang/Nim/issues/4906>`_) +- Fixed "startProcess changes working directory" + (`#4867 <https://github.com/nim-lang/Nim/issues/4867>`_) +- Fixed "bindsym to void template produces ICE" + (`#4808 <https://github.com/nim-lang/Nim/issues/4808>`_) +- Fixed "readline(TFile, var string) segfaults if second argument is nil" + (`#564 <https://github.com/nim-lang/Nim/issues/564>`_) +- Fixed "times.parse gives the wrong day of the week for the first hour of the day." + (`#4922 <https://github.com/nim-lang/Nim/issues/4922>`_) +- Fixed "Internal error when passing parameter proc inside .gcsafe closure" + (`#4927 <https://github.com/nim-lang/Nim/issues/4927>`_) +- Fixed "Upcoming asyncdispatch doesn't compile with C++ backend on OS X" + (`#4928 <https://github.com/nim-lang/Nim/issues/4928>`_) diff --git a/web/news/e029_version_0_16_0.rst b/web/news/e029_version_0_16_0.rst new file mode 100644 index 000000000..a6c8aa20f --- /dev/null +++ b/web/news/e029_version_0_16_0.rst @@ -0,0 +1,60 @@ +Version 0.16.0 released +======================= + +.. container:: metadata + + Posted by xyz on dd/mm/yyyy + +We're happy to announce that the latest release of Nim, version 0.16.0, is now +available! + +As always, you can grab the latest version from the +`downloads page <http://nim-lang.org/download.html>`_. + +This release includes almost xyz bug fixes and improvements. To see a full list +of changes, take a look at the detailed changelog +`below <#changelog>`_. + +Some of the most significant changes in this release include: xyz + + +Changelog +~~~~~~~~~ + +Changes affecting backwards compatibility +----------------------------------------- + +- ``staticExec`` now uses the directory of the nim file that contains the + ``staticExec`` call as the current working directory. +- ``TimeInfo.tzname`` has been removed from ``times`` module because it was + broken. Because of this, the option ``"ZZZ"`` will no longer work in format + strings for formatting and parsing. + +Library Additions +----------------- + +- Added new parameter to ``error`` proc of ``macro`` module to provide better + error message +- Added new ``deques`` module intended to replace ``queues``. + ``deques`` provides a superset of ``queues`` API with clear naming. + ``queues`` module is now deprecated and will be removed in the future. + +Tool Additions +-------------- + + +Compiler Additions +------------------ + + +Language Additions +------------------ + + +Bugfixes +-------- + +The list below has been generated based on the commits in Nim's git +repository. As such it lists only the issues which have been closed +via a commit, for a full list see +`this link on Github <https://github.com/nim-lang/Nim/issues?utf8=%E2%9C%93&q=is%3Aissue+closed%3A%222016-06-22+..+2016-09-30%22+>`_. diff --git a/web/news/e030_nim_in_action_in_production.rst b/web/news/e030_nim_in_action_in_production.rst new file mode 100644 index 000000000..b68b82801 --- /dev/null +++ b/web/news/e030_nim_in_action_in_production.rst @@ -0,0 +1,53 @@ +Nim in Action is going into production! +======================================= + +.. container:: metadata + + Posted by Dominik Picheta on 20/11/2016 + +.. raw::html + + <a href="https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81"> + <img src="../assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!" width="682"/> + </a> + +I am very happy to say that just last week I have put the finishing touches +on Nim in Action. The final manuscript has been submitted to Manning (the book's +publisher), and the printed version is expected to start shipping in March +2017 (give or take 1 month). + +The eBook is still available and now contains all of the book's chapters, +including new ones dealing with the foreign function interface and +metaprogramming. +That said, it may still take some time before the eBook is updated with the +latest corrections. + +I am incredibly thankful to everyone that purchased the book already. Many of +you have also given me a lot of `brilliant <http://forum.nim-lang.org/t/1978>`_ +`feedback <https://forums.manning.com/forums/nim-in-action>`_, +thank you very much for +taking the time to do so. I have done my best to act on this +feedback and I hope you will agree that the book has risen in quality as a +result. + +Writing this book has been both exhausting and incredible at the same time. +I look forward +to having a physical copy of it in my hands, and I'm sure many of you do as +well. I can safely say that without your support this book would not have +happened, even if you did not purchase a copy your interest in Nim has made it +possible and I thank you for that. + +As always, you can make a purchase on +`Manning's website <https://manning.com/books/nim-in-action?a_aid=niminaction&a_bid=78a27e81>`_. +Both eBook's and printed books are available, and purchasing a printed book will +get you an eBook for free. +You can now also pre-order Nim in Action on +`Amazon <https://www.amazon.co.uk/Nim-Action-Dominik-Picheta/dp/1617293431/ref=sr_1_1?ie=UTF8&qid=1479663850&sr=8-1&keywords=nim+in+action>`_! + +If you would like updates about the book then please feel free to +follow either `myself <https://twitter.com/d0m96>`_ or +`@nim_lang <https://twitter.com/nim_lang>`_ on Twitter. Finally, if you have any +questions, do get in touch via `Twitter, NimForum, +IRC or Gitter <http://nim-lang.org/community.html>`_. + +Thanks for reading! diff --git a/web/support.rst b/web/support.rst index 9a526605e..72e6dad71 100644 --- a/web/support.rst +++ b/web/support.rst @@ -32,9 +32,9 @@ Commercial support includes: All interested parties should email ``support@nim-lang.org``. The bid for contracting work is a commercial offer provided by: -| **METATEXX GmbH** -| Spicher Str. 30 -| 53859 Niederkassel + +| **Andreas Rumpf** +| St.-Quentin-Ring 47 +| 67663 Kaiserslautern | GERMANY -| EU VAT-IN: DE287088604 -| http://metatexx.de/index.php?index=12 +| EU VAT-IN: DE297783450 diff --git a/web/ticker.html b/web/ticker.html index a05ea8476..86dc97c14 100644 --- a/web/ticker.html +++ b/web/ticker.html @@ -1,26 +1,26 @@ -<a class="news" href="$1news/2016_09_30_version_0_15_0_released.html"> +<a class="news" href="$1news/e030_nim_in_action_in_production.html"> + <h4>November 20, 2016</h4> + <p>Nim in Action is going into production!</p> +</a> + +<a class="news" href="$1news/e028_version_0_15_2.html"> + <h4>October 23, 2016</h4> + <p>Nim version 0.15.2 has been released!</p> +</a> + +<a class="news" href="$1news/e027_version_0_15_0.html"> <h4>September 30, 2016</h4> <p>Nim version 0.15.0 has been released!</p> </a> -<a class="news" href="$1news/2016_09_03_nim_community_survey_results.html"> +<a class="news" href="$1news/e026_survey_results.html"> <h4>September 3, 2016</h4> <p>Nim Community Survey results</p> </a> -<a class="news" href="$1news/2016_08_06_bountysource_update_the_road_to_v10.html"> +<a class="news" href="$1news/e025_bountysource_update.html"> <h4>August 6, 2016</h4> <p>BountySource Update: The Road to v1.0</p> </a> -<a class="news" href="$1news/2016_06_23_launching_the_2016_nim_community_survey.html"> - <h4>June 23, 2016</h4> - <p>Launching the 2016 Nim community survey!</p> -</a> - -<a class="news" href="$1news/2016_06_11_version_0_14_2_released.html"> - <h4>June 11, 2016</h4> - <p>Nim version 0.14.2 has been released!</p> -</a> - <a href="$1news.html" class="blue">See All News...</a> diff --git a/web/website.ini b/web/website.ini index 860ab9338..3b8203cc0 100644 --- a/web/website.ini +++ b/web/website.ini @@ -31,7 +31,7 @@ file: ticker.html [Documentation] doc: "endb;intern;apis;lib;manual.rst;tut1.rst;tut2.rst;nimc;overview;filters" doc: "tools;niminst;nimgrep;gc;estp;idetools;docgen;koch;backends.rst" -doc: "nimfix.rst;nimsuggest.rst;nep1.rst;nims.rst" +doc: "nimfix.rst;nimsuggest.rst;nep1.rst;nims.rst;contributing.rst" pdf: "manual.rst;lib.rst;tut1.rst;tut2.rst;nimc.rst;niminst.rst;gc.rst" srcdoc2: "system.nim;system/nimscript;pure/ospaths" srcdoc2: "core/macros;pure/marshal;core/typeinfo" @@ -45,13 +45,13 @@ srcdoc2: "pure/parseopt;pure/parseopt2;pure/hashes;pure/strtabs;pure/lexbase" srcdoc2: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql" srcdoc2: "pure/streams;pure/terminal;pure/cgi;pure/unicode;pure/strmisc" srcdoc2: "pure/htmlgen;pure/parseutils;pure/browsers" -srcdoc2: "impure/db_postgres;impure/db_mysql;impure/db_sqlite" +srcdoc2: "impure/db_postgres;impure/db_mysql;impure/db_sqlite;pure/db_common" srcdoc2: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl;pure/fsmonitor" srcdoc2: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser" srcdoc2: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes" srcdoc2: "pure/json;pure/base64;pure/scgi" srcdoc2: "pure/collections/tables;pure/collections/sets;pure/collections/lists" -srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/encodings" +srcdoc2: "pure/collections/intsets;pure/collections/queues;pure/collections/deques;pure/encodings" srcdoc2: "pure/events;pure/collections/sequtils;pure/cookies" srcdoc2: "pure/memfiles;pure/subexes;pure/collections/critbits" srcdoc2: "deprecated/pure/asyncio;deprecated/pure/actors;core/locks;core/rlocks;pure/oids;pure/endians;pure/uri" @@ -63,7 +63,7 @@ srcdoc2: "deprecated/pure/ftpclient" srcdoc2: "pure/asyncfile;pure/asyncftpclient" srcdoc2: "pure/md5;pure/rationals" srcdoc2: "posix/posix" -srcdoc2: "pure/fenv;pure/securehash" +srcdoc2: "pure/fenv;pure/securehash;impure/rdstdin" srcdoc2: "pure/basic2d;pure/basic3d;pure/mersenne;pure/coro;pure/httpcore" ; Note: everything under 'webdoc' doesn't get listed in the index, so wrappers |
