diff options
84 files changed, 2507 insertions, 487 deletions
diff --git a/.gitignore b/.gitignore index 38b797612..9de4569bf 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,9 @@ dnimcache/ !/icons/*.o *.obj *.ilk +*.exp *.pdb +*.lib *.dll *.exe *.so @@ -19,7 +21,6 @@ dnimcache/ *.zip *.iss *.log -*.ilk *.pdb mapping.txt @@ -42,6 +43,7 @@ bin/* *.perspectivev3 *.swp .DS_Store +.tags project.xcworkspace/ xcuserdata/ diff --git a/changelog.md b/changelog.md index 7f6ad1ecb..297a6c87b 100644 --- a/changelog.md +++ b/changelog.md @@ -136,6 +136,9 @@ proc enumToString*(enums: openArray[enum]): string = slightly. The `dumpLisp` macro in this module now outputs an indented proper Lisp, devoid of commas. +- Added `macros.signatureHash` that returns a stable identifier + derived from the signature of a symbol. + - In `strutils` empty strings now no longer matched as substrings anymore. @@ -192,6 +195,11 @@ proc enumToString*(enums: openArray[enum]): string = ### Compiler changes - The deprecated `fmod` proc is now unavailable on the VM'. - +- A new `--outdir` option was added. +- The compiled JavaScript file for the project produced by executing `nim js` + will no longer be placed in the nimcache directory. +- The `--hotCodeReloading` has been implemented for the native targets. + The compiler also provides a new more flexible API for handling the + hot code reloading events in the code. ### Bugfixes diff --git a/compiler/ast.nim b/compiler/ast.nim index e691cc175..607b497fb 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -227,7 +227,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # already 33 flags! + TSymFlag* = enum # already 34 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -276,6 +276,8 @@ type # the calling side of the macro, not from the # implementation. sfGenSym # symbol is 'gensym'ed; do not add to symbol table + sfNonReloadable # symbol will be left as-is when hot code reloading is on - + # meaning that it won't be renamed and/or changed in any way TSymFlags* = set[TSymFlag] @@ -468,6 +470,7 @@ type nfDefaultParam # an automatically inserter default parameter nfDefaultRefsParam # a default param value references another parameter # the flag is applied to proc default values and to calls + nfExecuteOnReload # A top-level statement that will be executed during reloads TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: beyond that) @@ -654,7 +657,7 @@ type mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, - mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, + mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNBindSym, mLocals, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, @@ -854,7 +857,7 @@ type offset*: int # offset of record field loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a - # reference to another object to safe space) + # reference to another object to save space) constraint*: PNode # additional constraints like 'lit|result'; also # misused for the codegenDecl pragma in the hope # it won't cause problems @@ -978,7 +981,8 @@ const PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfPreventCg, nfLL, - nfFromTemplate, nfDefaultRefsParam} + nfFromTemplate, nfDefaultRefsParam, + nfExecuteOnReload} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index d177e1f88..4c8fa7147 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -185,7 +185,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) = var length = sonsLen(ri) for i in countup(1, length - 1): genParamLoop(params) - fixupCall(p, le, ri, d, op.r, params) + fixupCall(p, le, ri, d, rdLoc(op), params) proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = @@ -209,7 +209,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = genParamLoop(pl) template genCallPattern {.dirty.} = - lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc]) + lineF(p, cpsStmts, callPattern & ";$n", [rdLoc(op), pl, pl.addComma, rawProc]) let rawProc = getRawProcType(p, typ) let callPattern = if tfIterator in typ.flags: PatIter else: PatProc @@ -237,7 +237,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = assert(d.t != nil) # generate an assignment to d: var list: TLoc initLoc(list, locCall, d.lode, OnUnknown) - list.r = callPattern % [op.r, pl, pl.addComma, rawProc] + list.r = callPattern % [rdLoc(op), pl, pl.addComma, rawProc] genAssignment(p, d, list, {}) # no need for deep copying else: genCallPattern() diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f92f2e4de..a40c60e6d 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1534,6 +1534,7 @@ proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = var a: TLoc initLocExpr(p, n.sons[1], a) a.r = ropecg(p.module, frmt, [rdLoc(a)]) + a.flags = a.flags - {lfIndirect} # this flag should not be propagated here (not just for HCR) if d.k == locNone: getTemp(p, n.typ, d) genAssignment(p, d, a, {}) gcUsage(p.config, n) @@ -2034,8 +2035,28 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = genCall(p, e, d) of mNewString, mNewStringOfCap, mExit, mParseBiggestFloat: var opr = e.sons[0].sym + # Why would anyone want to set nodecl to one of these hardcoded magics? + # - not sure, and it wouldn't work if the symbol behind the magic isn't + # somehow forward-declared from some other usage, but it is *possible* if lfNoDecl notin opr.loc.flags: + let prc = magicsys.getCompilerProc(p.module.g.graph, $opr.loc.r) + # HACK: + # Explicitly add this proc as declared here so the cgsym call doesn't + # add a forward declaration - without this we could end up with the same + # 2 forward declarations. That happens because the magic symbol and the original + # one that shall be used have different ids (even though a call to one is + # actually a call to the other) so checking into m.declaredProtos with the 2 different ids doesn't work. + # Why would 2 identical forward declarations be a problem? + # - in the case of hot code-reloading we generate function pointers instead + # of forward declarations and in C++ it is an error to redefine a global + let wasDeclared = containsOrIncl(p.module.declaredProtos, prc.id) + # Make the function behind the magic get actually generated - this will + # not lead to a forward declaration! The genCall will lead to one. discard cgsym(p.module, $opr.loc.r) + # make sure we have pointer-initialising code for hot code reloading + if not wasDeclared and p.hcrOn: + addf(p.module.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [mangleDynLibProc(prc), getTypeDesc(p.module, prc.loc.t), getModuleDllPath(p.module, prc)]) genCall(p, e, d) of mReset: genReset(p, e) of mEcho: genEcho(p, e[1].skipConv) @@ -2292,6 +2313,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = proc expr(p: BProc, n: PNode, d: var TLoc) = p.currLineInfo = n.info + case n.kind of nkSym: var sym = n.sym diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 2d68a198e..ccb5a7635 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -31,7 +31,7 @@ const cfsData: "NIM_merge_DATA", cfsProcs: "NIM_merge_PROCS", cfsInitProc: "NIM_merge_INIT_PROC", - cfsDatInitProc: "NIM_merge_DATINIT_PROC", + cfsDatInitProc: "NIM_merge_DATINIT_PROC", cfsTypeInit1: "NIM_merge_TYPE_INIT1", cfsTypeInit2: "NIM_merge_TYPE_INIT2", cfsTypeInit3: "NIM_merge_TYPE_INIT3", diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index bc8735397..6dc10db3d 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -15,18 +15,20 @@ const stringCaseThreshold = 8 # above X strings a hash-switch for strings is generated -proc registerGcRoot(p: BProc, v: PSym) = +proc getTraverseProc(p: BProc, v: Psym): Rope = if p.config.selectedGC in {gcMarkAndSweep, gcDestructors, gcV2, gcRefc} and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) - let prc = genTraverseProcForGlobal(p.module, v, v.info) - if sfThread in v.flags: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterThreadLocalMarker($1);$n", [prc]) - else: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterGlobalMarker($1);$n", [prc]) + result = genTraverseProcForGlobal(p.module, v, v.info) + +proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) = + if sfThread in v.flags: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc]) + else: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc]) proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false @@ -41,6 +43,10 @@ proc inExceptBlockLen(p: BProc): int = for x in p.nestedTryStmts: if x.inExcept: result.inc +proc startBlock(p: BProc, start: FormatStr = "{$n", + args: varargs[Rope]): int {.discardable.} +proc endBlock(p: BProc) + proc genVarTuple(p: BProc, n: PNode) = var tup, field: TLoc if n.kind != nkVarTuple: internalError(p.config, n.info, "genVarTuple") @@ -52,6 +58,32 @@ proc genVarTuple(p: BProc, n: PNode) = genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc)) return + # check only the first son + var forHcr = treatGlobalDifferentlyForHCR(p.module, n.sons[0].sym) + let hcrCond = if forHcr: getTempName(p.module) else: nil + var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]] + # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block) + let isGlobalInBlock = forHcr and p.blocks.len > 2 + # do not close and reopen blocks if this is a 'global' but inside of a block (if/while/block) + forHcr = forHcr and not isGlobalInBlock + + if forHcr: + # check with the boolean if the initializing code for the tuple should be ran + lineCg(p, cpsStmts, "if ($1)$n", hcrCond) + startBlock(p) + defer: + if forHcr: + # end the block where the tuple gets initialized + endBlock(p) + if forHcr or isGlobalInBlock: + # insert the registration of the globals for the different parts of the tuple at the + # start of the current scope (after they have been iterated) and init a boolean to + # check if any of them is newly introduced and the initializing code has to be ran + lineCg(p, cpsLocals, "NIM_BOOL $1 = NIM_FALSE;$n", hcrCond) + for curr in hcrGlobals: + lineCg(p, cpsLocals, "$1 |= hcrRegisterGlobal($4, \"$2\", sizeof($3), $5, (void**)&$2);$N", + hcrCond, curr.loc.r, rdLoc(curr.loc), getModuleDllPath(p.module, n.sons[0].sym), curr.tp) + genLineDir(p, n) initLocExpr(p, n.sons[L-1], tup) var t = tup.t.skipTypes(abstractInst) @@ -59,10 +91,13 @@ proc genVarTuple(p: BProc, n: PNode) = let vn = n.sons[i] let v = vn.sym if sfCompileTime in v.flags: continue + var traverseProc: Rope if sfGlobal in v.flags: assignGlobalVar(p, vn) genObjectInit(p, cpsInit, v.typ, v.loc, true) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: assignLocalVar(p, vn) initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[L-1])) @@ -73,6 +108,8 @@ proc genVarTuple(p: BProc, n: PNode) = if t.n.sons[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple") field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym)] putLocIntoDest(p, v.loc, field) + if forHcr or isGlobalInBlock: + hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc)) proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) @@ -242,6 +279,7 @@ proc genSingleVar(p: BProc, a: PNode) = genGotoVar(p, a.sons[2]) return var targetProc = p + var traverseProc: Rope if sfGlobal in v.flags: if v.flags * {sfImportc, sfExportc} == {sfImportc} and a.sons[2].kind == nkEmpty and @@ -270,7 +308,9 @@ proc genSingleVar(p: BProc, a: PNode) = # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc) if sfExportc in v.flags and p.module.g.generatedHeader != nil: genVarPrototype(p.module.g.generatedHeader, vn) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: let value = a.sons[2] let imm = isAssignedImmediately(p.config, value) @@ -302,6 +342,30 @@ proc genSingleVar(p: BProc, a: PNode) = assignLocalVar(p, vn) initLocalVar(p, v, imm) + if traverseProc == nil: traverseProc = ~"NULL" + # If the var is in a block (control flow like if/while or a block) in global scope just + # register the so called "global" so it can be used later on. There is no need to close + # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway. + var forHcr = treatGlobalDifferentlyForHCR(p.module, v) + if forHcr and targetProc.blocks.len > 3 and v.owner.kind == skModule: + # put it in the locals section - mainly because of loops which + # use the var in a call to resetLoc() in the statements section + lineCg(targetProc, cpsLocals, "hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1);$n", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + # nothing special left to do later on - let's avoid closing and reopening blocks + forHcr = false + + # we close and reopen the global if (nim_hcr_do_init_) blocks in the main Init function + # for the module so we can have globals and top-level code be interleaved and still + # be able to re-run it but without the top level code - just the init of globals + if forHcr: + lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + startBlock(targetProc) + defer: + if forHcr: + endBlock(targetProc) + if a.sons[2].kind != nkEmpty: genLineDir(targetProc, a) loadInto(targetProc, a.sons[0], a.sons[2], v.loc) diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 0a2bbf93b..87e7c9d48 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -134,11 +134,13 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = var c: TTraversalClosure var p = newProc(nil, m) result = "Marker_" & getTypeName(m, origTyp, sig) - var typ = origTyp.skipTypes(abstractInstOwned) + let + hcrOn = m.hcrOn + typ = origTyp.skipTypes(abstractInstOwned) + markerName = if hcrOn: result & "_actual" else: result + header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [markerName] + t = getTypeDesc(m, typ) - let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result] - - let t = getTypeDesc(m, typ) lineF(p, cpsLocals, "$1 a;$n", [t]) lineF(p, cpsInit, "a = ($1)p;$n", [t]) @@ -155,18 +157,23 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = else: genTraverseProc(c, "(*a)".rope, typ.sons[0]) - let generatedProc = "$1 {$n$2$3$4}$n" % + let generatedProc = "$1 {$n$2$3$4}\n" % [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)] - m.s[cfsProcHeaders].addf("$1;$n", [header]) + m.s[cfsProcHeaders].addf("$1;\n", [header]) m.s[cfsProcs].add(generatedProc) + if hcrOn: + addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result]) + addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n", + [result, markerName, getModuleDllPath(m)]) + proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope = discard genTypeInfo(m, s.loc.t, info) var c: TTraversalClosure var p = newProc(nil, m) - var sLoc = s.loc.r + var sLoc = rdLoc(s.loc) result = getTempName(m) if sfThread in s.flags and emulatedThreadVars(m.config): diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index afe90544d..063c02df9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -14,7 +14,7 @@ import sighashes from lowerings import createObj -proc genProcHeader(m: BModule, prc: PSym): Rope +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope proc isKeyword(w: PIdent): bool = # Nim and C++ share some keywords @@ -59,7 +59,23 @@ proc mangleParamName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: var res = s.name.s.mangle - if isKeyword(s.name) or m.g.config.cppDefines.contains(res): + # Take into account if HCR is on because of the following scenario: + # if a module gets imported and it has some more importc symbols in it, + # some param names might recieve the "_0" suffix to distinguish from what + # is newly available. That might lead to changes in the C code in nimcache + # that contain only a parameter name change, but that is enough to mandate + # recompilation of that source file and thus a new shared object will be + # relinked. That may lead to a module getting reloaded which wasn't intended + # and that may be fatal when parts of the current active callstack when + # performCodeReload() was called are from the module being reloaded + # unintentionally - example (3 modules which import one another): + # main => proxy => reloadable + # we call performCodeReload() in proxy to reload only changes in reloadable + # but there is a new import which introduces an importc symbol `socket` + # and a function called in main or proxy uses `socket` as a parameter name. + # That would lead to either needing to reload `proxy` or to overwrite the + # executable file for the main module, which is running (or both!) -> error. + if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): res.add "_0" result = res.rope s.loc.r = result @@ -507,6 +523,8 @@ proc fillObjectFields*(m: BModule; typ: PType) = var check = initIntSet() discard getRecordFields(m, typ, check) +proc mangleDynLibProc(sym: PSym): Rope + proc getRecordDesc(m: BModule, typ: PType, name: Rope, check: var IntSet): Rope = # declare the record: @@ -535,22 +553,13 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, appcg(m, result, " : public $1 {$n", [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)]) if typ.isException: - appcg(m, result, "virtual void raise() {throw *this;}$n") # required for polymorphic exceptions + appcg(m, result, "virtual void raise() { throw *this; }$n") # required for polymorphic exceptions if typ.sym.magic == mException: # Add cleanup destructor to Exception base class - appcg(m, result, "~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) - # hack: forward declare popCurrentExceptionEx() on top of type description, - # proper request to generate popCurrentExceptionEx not possible for 2 reasons: - # generated function will be below declared Exception type and circular dependency - # between Exception and popCurrentExceptionEx function - - let popExSym = magicsys.getCompilerProc(m.g.graph, "popCurrentExceptionEx") - if lfDynamicLib in popExSym.loc.flags and sfImportc in popExSym.flags: - # echo popExSym.flags, " ma flags ", popExSym.loc.flags - result = "extern " & getTypeDescAux(m, popExSym.typ, check) & " " & - mangleName(m, popExSym) & ";\L" & result - else: - result = genProcHeader(m, popExSym) & ";\L" & result + appcg(m, result, "~$1();$n", [name]) + # define it out of the class body and into the procs section so we don't have to + # artificially forward-declare popCurrentExceptionEx (very VERY troublesome for HCR) + appcg(m, cfsProcs, "inline $1::~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) hasField = true else: appcg(m, result, " {$n $1 Sup;$n", @@ -888,31 +897,43 @@ proc finishTypeDescriptions(m: BModule) = template cgDeclFrmt*(s: PSym): string = s.constraint.strVal -proc genProcHeader(m: BModule, prc: PSym): Rope = +proc isReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable notin prc.flags + +proc isNonReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable in prc.flags + +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope = var rettype, params: Rope - genCLineDir(result, prc.info, m.config) # using static is needed for inline procs if lfExportLib in prc.loc.flags: if isHeaderFile in m.flags: result.add "N_LIB_IMPORT " else: result.add "N_LIB_EXPORT " - elif prc.typ.callConv == ccInline: + elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc): result.add "static " elif {sfImportc, sfExportc} * prc.flags == {}: result.add "N_LIB_PRIVATE " var check = initIntSet() fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) genProcParams(m, prc.typ, rettype, params, check) + # handle the 2 options for hotcodereloading codegen - function pointer + # (instead of forward declaration) or header for function budy with "_actual" postfix + let asPtrStr = rope(if asPtr: "_PTR" else: "") + var name = prc.loc.r + if isReloadable(m, prc) and not asPtr: + add(name, "_actual") # careful here! don't access ``prc.ast`` as that could reload large parts of # the object graph! if prc.constraint.isNil: - addf(result, "$1($2, $3)$4", - [rope(CallingConvToStr[prc.typ.callConv]), rettype, prc.loc.r, + addf(result, "$1$2($3, $4)$5", + [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name, params]) else: - result = prc.cgDeclFrmt % [rettype, prc.loc.r, params] + let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name + result = prc.cgDeclFrmt % [rettype, asPtrStr, params] # ------------------ type info generation ------------------------------------- @@ -921,6 +942,9 @@ proc getNimNode(m: BModule): Rope = result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)] inc(m.typeNodes) +proc TINameForHcr(m: BModule, name: Rope): Rope = + return if m.hcrOn: "(*".rope & name & ")" else: name + proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope; info: TLineInfo) = var nimtypeKind: int @@ -930,19 +954,21 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; else: nimtypeKind = ord(typ.kind) + let nameHcr = TINameForHcr(m, name) + var size: Rope if tfIncompleteStruct in typ.flags: size = rope"void*" else: size = getTypeDesc(m, origType) addf(m.s[cfsTypeInit3], "$1.size = sizeof($2);$n" & "$1.kind = $3;$n" & "$1.base = $4;$n", - [name, size, rope(nimtypeKind), base]) + [nameHcr, size, rope(nimtypeKind), base]) # compute type flags for GC optimization var flags = 0 if not containsGarbageCollectedRef(typ): flags = flags or 1 if not canFormAcycle(typ): flags = flags or 2 #else MessageOut("can contain a cycle: " & typeToString(typ)) if flags != 0: - addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) + addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [nameHcr, rope(flags)]) discard cgsym(m, "TNimType") if isDefined(m.config, "nimTypeNames"): var typename = typeToString(if origType.typeInst != nil: origType.typeInst @@ -950,11 +976,17 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", - [name, makeCstring typename]) + [nameHcr, makeCstring typename]) discard cgsym(m, "nimTypeRoot") addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n", - [name]) - addf(m.s[cfsVars], "TNimType $1;$n", [name]) + [nameHcr]) + + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n", + [name, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsVars], "TNimType $1;$n", [name]) proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) = @@ -984,6 +1016,14 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope = var tmp = discriminatorTableName(m, objtype, d) result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)] +proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimNode** $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($3, \"$1\", sizeof(TNimNode*) * $2, NULL, (void**)&$1);$n", + [name, size, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [name, size]) + proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; info: TLineInfo) = case n.kind @@ -992,8 +1032,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; if L == 1: genObjectFields(m, typ, origType, n.sons[0], expr, info) elif L > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)]) + var tmp = getTempName(m) & "_" & $L + genTNimNodeArray(m, tmp, rope(L)) for i in countup(0, L-1): var tmp2 = getNimNode(m) addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2]) @@ -1066,7 +1106,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo var tmp = getNimNode(m) if not isImportedType(typ): genObjectFields(m, typ, origType, typ.n, tmp, info) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), tmp]) var t = typ.sons[0] while t != nil: t = t.skipTypes(skipPtrs) @@ -1078,8 +1118,8 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) var expr = getNimNode(m) var length = sonsLen(typ) if length > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)]) + var tmp = getTempName(m) & "_" & $length + genTNimNodeArray(m, tmp, rope(length)) for i in countup(0, length - 1): var a = typ.sons[i] var tmp2 = getNimNode(m) @@ -1094,7 +1134,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) else: addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2;$n", [expr, rope(length)]) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), expr]) proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # Type information for enumerations is quite heavy, so we do some @@ -1102,10 +1142,9 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # anyway. We generate a cstring array and a loop over it. Exceptional # positions will be reset after the loop. genTypeInfoAux(m, typ, typ, name, info) - var nodePtrs = getTempName(m) var length = sonsLen(typ.n) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", - [nodePtrs, rope(length)]) + var nodePtrs = getTempName(m) & "_" & $length + genTNimNodeArray(m, nodePtrs, rope(length)) var enumNames, specialCases: Rope var firstNimNode = m.typeNodes var hasHoles = false @@ -1134,17 +1173,17 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = add(m.s[cfsTypeInit3], specialCases) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n$4.node = &$1;$n", - [getNimNode(m), rope(length), nodePtrs, name]) + [getNimNode(m), rope(length), nodePtrs, TINameForHcr(m, name)]) if hasHoles: # 1 << 2 is {ntfEnumHole} - addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name]) + addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [TINameForHcr(m, name)]) proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = assert(typ.sons[0] != nil) genTypeInfoAux(m, typ, typ, name, info) var tmp = getNimNode(m) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n", - [tmp, rope(firstOrd(m.config, typ)), name]) + [tmp, rope(firstOrd(m.config, typ)), TINameForHcr(m, name)]) proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info) @@ -1169,19 +1208,29 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = let origType = t var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses) + let prefixTI = if m.hcrOn: "(" else: "(&" + let sig = hashType(origType) result = m.typeInfoMarker.getOrDefault(sig) if result != nil: - return "(&".rope & result & ")".rope + return prefixTI.rope & result & ")".rope - result = m.g.typeInfoMarker.getOrDefault(sig) - if result != nil: + proc declareNimType(m: BModule, str: Rope, ownerModule: PSym) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [str]) + addf(m.s[cfsTypeInit1], "\t$1 = (TNimType*)hcrGetGlobal($2, \"$1\");$n", + [str, getModuleDllPath(m, ownerModule)]) + else: + addf(m.s[cfsVars], "extern TNimType $1;$n", [str]) + + let marker = m.g.typeInfoMarker.getOrDefault(sig) + if marker.str != nil: discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) + declareNimType(m, marker.str, marker.owner) # also store in local type section: - m.typeInfoMarker[sig] = result - return "(&".rope & result & ")".rope + m.typeInfoMarker[sig] = marker.str + return prefixTI.rope & marker.str & ")".rope result = "NTI$1_" % [rope($sig)] m.typeInfoMarker[sig] = result @@ -1194,10 +1243,10 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = # reference the type info as extern here discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) - return "(&".rope & result & ")".rope + declareNimType(m, result, owner) + return prefixTI.rope & result & ")".rope - m.g.typeInfoMarker[sig] = result + m.g.typeInfoMarker[sig] = (str: result, owner: owner) case t.kind of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent: @@ -1219,12 +1268,12 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = if m.config.selectedGC != gcDestructors: if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyRef: genTypeInfoAux(m, t, t, result, info) if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info) of tyArray: genArrayInfo(m, t, result, info) of tySet: genSetInfo(m, t, result, info) @@ -1241,7 +1290,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = genDeepCopyProc(m, t.deepCopy, result) elif origType.deepCopy != nil: genDeepCopyProc(m, origType.deepCopy, result) - result = "(&".rope & result & ")".rope + result = prefixTI.rope & result & ")".rope proc genTypeSection(m: BModule, n: PNode) = discard diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cb186de10..d8f426f05 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,8 +16,6 @@ import condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf -import system/indexerrors - when not defined(leanCompiler): import semparallel @@ -46,6 +44,9 @@ when options.hasTinyCBackend: # implementation +proc hcrOn(m: BModule): bool = m.config.hcrOn +proc hcrOn(p: BProc): bool = p.module.config.hcrOn + proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) @@ -92,6 +93,17 @@ proc useHeader(m: BModule, sym: PSym) = proc cgsym(m: BModule, name: string): Rope +proc getCFile(m: BModule): AbsoluteFile + +proc getModuleDllPath(m: BModule): Rope = + let (dir, name, ext) = splitFile(getCFile(m)) + let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext]) + return makeCString(dir.string & "/" & filename) + +proc getModuleDllPath(m: BModule, s: PSym): Rope = + return getModuleDllPath(findPendingModule(m, s)) + +# TODO: please document proc ropecg(m: BModule, frmt: FormatStr, args: varargs[Rope]): Rope = assert m != nil var i = 0 @@ -440,12 +452,18 @@ proc assignLocalVar(p: BProc, n: PNode) = include ccgthreadvars proc varInDynamicLib(m: BModule, sym: PSym) -proc mangleDynLibProc(sym: PSym): Rope + +proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool = + return m.hcrOn and {sfThread, sfGlobal} * s.flags == {sfGlobal} and + ({lfNoDecl, lfHeader} * s.loc.flags == {}) + # and s.owner.kind == skModule # owner isn't always a module (global pragma on local var) + # and s.loc.k == locGlobalVar # loc isn't always initialized when this proc is used proc assignGlobalVar(p: BProc, n: PNode) = let s = n.sym if s.loc.k == locNone: fillLoc(s.loc, locGlobalVar, n, mangleName(p.module, s), OnHeap) + if treatGlobalDifferentlyForHCR(p.module, s): incl(s.loc.flags, lfIndirect) if lfDynamicLib in s.loc.flags: var q = findPendingModule(p.module, s) @@ -463,8 +481,10 @@ proc assignGlobalVar(p: BProc, n: PNode) = var decl: Rope = nil var td = getTypeDesc(p.module, s.loc.t) if s.constraint.isNil: - if sfImportc in s.flags: add(decl, "extern ") + if p.hcrOn: add(decl, "static ") + elif sfImportc in s.flags: add(decl, "extern ") add(decl, td) + if p.hcrOn: add(decl, "*") if sfRegister in s.flags: add(decl, " register") if sfVolatile in s.flags: add(decl, " volatile") addf(decl, " $1;$n", [s.loc.r]) @@ -853,6 +873,14 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum = for i in 0..<safeLen(n): allPathsInBranch(n[i]) +proc getProcTypeCast(m: BModule, prc: PSym): Rope = + result = getTypeDesc(m, prc.loc.t) + if prc.typ.callConv == ccClosure: + var rettype, params: Rope + var check = initIntSet() + genProcParams(m, prc.typ, rettype, params, check) + result = "$1(*)$2" % [rettype, params] + proc genProcAux(m: BModule, prc: PSym) = var p = newProc(prc, m) var header = genProcHeader(m, prc) @@ -931,6 +959,9 @@ proc genProcAux(m: BModule, prc: PSym) = add(generatedProc, returnStmt) add(generatedProc, ~"}$N") add(m.s[cfsProcs], generatedProc) + if isReloadable(m, prc): + addf(m.s[cfsDynLibInit], "\t$1 = ($3) hcrRegisterProc($4, \"$1\", (void*)$2);$n", + [prc.loc.r, prc.loc.r & "_actual", getProcTypeCast(m, prc), getModuleDllPath(m, prc)]) proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} = result = (sfCompileToCpp in m.module.flags and @@ -946,20 +977,27 @@ proc genProcPrototype(m: BModule, sym: PSym) = if lfDynamicLib in sym.loc.flags: if getModule(sym).id != m.module.id and not containsOrIncl(m.declaredThings, sym.id): - add(m.s[cfsVars], ropecg(m, "extern $1 $2;$n", + add(m.s[cfsVars], ropecg(m, "$1 $2 $3;$n", + rope(if isReloadable(m, sym): "static" else: "extern"), getTypeDesc(m, sym.loc.t), mangleDynLibProc(sym))) + if isReloadable(m, sym): + addf(m.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [mangleDynLibProc(sym), getTypeDesc(m, sym.loc.t), getModuleDllPath(m, sym)]) elif not containsOrIncl(m.declaredProtos, sym.id): - var header = genProcHeader(m, sym) - if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props: - header = "__declspec(noreturn) " & header - if sym.typ.callConv != ccInline and requiresExternC(m, sym): - header = "extern \"C\" " & header - if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props: - header.add(" __attribute__((naked))") - if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props: - header.add(" __attribute__((noreturn))") - add(m.s[cfsProcHeaders], ropecg(m, "$1;$n", header)) - + let asPtr = isReloadable(m, sym) + var header = genProcHeader(m, sym, asPtr) + if not asPtr: + if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props: + header = "__declspec(noreturn) " & header + if sym.typ.callConv != ccInline and requiresExternC(m, sym): + header = "extern \"C\" " & header + if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props: + header.add(" __attribute__((naked))") + if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props: + header.add(" __attribute__((noreturn))") + add(m.s[cfsProcHeaders], ropecg(m, "$1;$N", header)) + +# TODO: figure out how to rename this - it DOES generate a forward declaration proc genProcNoForward(m: BModule, prc: PSym) = if lfImportCompilerProc in prc.loc.flags: fillProcLoc(m, prc.ast[namePos]) @@ -969,7 +1007,6 @@ proc genProcNoForward(m: BModule, prc: PSym) = return if lfNoDecl in prc.loc.flags: fillProcLoc(m, prc.ast[namePos]) - useHeader(m, prc) genProcPrototype(m, prc) elif prc.typ.callConv == ccInline: # We add inline procs to the calling module to enable C based inlining. @@ -977,30 +1014,45 @@ proc genProcNoForward(m: BModule, prc: PSym) = # a check for ``m.declaredThings``. if not containsOrIncl(m.declaredThings, prc.id): #if prc.loc.k == locNone: - fillProcLoc(m, prc.ast[namePos]) + # mangle the inline proc based on the module where it is defined - not on the first module that uses it + fillProcLoc(findPendingModule(m, prc), prc.ast[namePos]) #elif {sfExportc, sfImportc} * prc.flags == {}: # # reset name to restore consistency in case of hashing collisions: # echo "resetting ", prc.id, " by ", m.module.name.s # prc.loc.r = nil # prc.loc.r = mangleName(m, prc) - useHeader(m, prc) genProcPrototype(m, prc) genProcAux(m, prc) elif lfDynamicLib in prc.loc.flags: var q = findPendingModule(m, prc) fillProcLoc(q, prc.ast[namePos]) - useHeader(m, prc) genProcPrototype(m, prc) if q != nil and not containsOrIncl(q.declaredThings, prc.id): symInDynamicLib(q, prc) + # register the procedure even though it is in a different dynamic library and will not be + # reloadable (and has no _actual suffix) - other modules will need to be able to get it through + # the hcr dynlib (also put it in the DynLibInit section - right after it gets loaded) + if isReloadable(q, prc): + addf(q.s[cfsDynLibInit], "\t$1 = ($2) hcrRegisterProc($3, \"$1\", (void*)$1);$n", + [prc.loc.r, getTypeDesc(q, prc.loc.t), getModuleDllPath(m, q.module)]) else: symInDynamicLibPartial(m, prc) elif sfImportc notin prc.flags: var q = findPendingModule(m, prc) fillProcLoc(q, prc.ast[namePos]) - useHeader(m, prc) + # generate a getProc call to initialize the pointer for this + # externally-to-the-current-module defined proc, also important + # to do the declaredProtos check before the call to genProcPrototype + if isReloadable(m, prc) and prc.id notin m.declaredProtos and + q != nil and q.module.id != m.module.id: + addf(m.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [prc.loc.r, getProcTypeCast(m, prc), getModuleDllPath(m, prc)]) genProcPrototype(m, prc) if q != nil and not containsOrIncl(q.declaredThings, prc.id): + # make sure there is a "prototype" in the external module + # which will actually become a function pointer + if isReloadable(m, prc): + genProcPrototype(q, prc) genProcAux(q, prc) else: fillProcLoc(m, prc.ast[namePos]) @@ -1049,6 +1101,8 @@ proc genVarPrototype(m: BModule, n: PNode) = let sym = n.sym useHeader(m, sym) fillLoc(sym.loc, locGlobalVar, n, mangleName(m, sym), OnHeap) + if treatGlobalDifferentlyForHCR(m, sym): incl(sym.loc.flags, lfIndirect) + if (lfNoDecl in sym.loc.flags) or contains(m.declaredThings, sym.id): return if sym.owner.id != m.module.id: @@ -1057,12 +1111,17 @@ proc genVarPrototype(m: BModule, n: PNode) = if sfThread in sym.flags: declareThreadVar(m, sym, true) else: - add(m.s[cfsVars], "extern ") + incl(m.declaredThings, sym.id) + add(m.s[cfsVars], if m.hcrOn: "static " else: "extern ") add(m.s[cfsVars], getTypeDesc(m, sym.loc.t)) + if m.hcrOn: add(m.s[cfsVars], "*") if lfDynamicLib in sym.loc.flags: add(m.s[cfsVars], "*") if sfRegister in sym.flags: add(m.s[cfsVars], " register") if sfVolatile in sym.flags: add(m.s[cfsVars], " volatile") addf(m.s[cfsVars], " $1;$n", [sym.loc.r]) + if m.hcrOn: addf(m.initProc.procSec(cpsLocals), + "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r, + getTypeDesc(m, sym.loc.t), getModuleDllPath(m, sym)]) const frameDefines = """ @@ -1106,22 +1165,76 @@ proc getCopyright(conf: ConfigRef; cfile: Cfile): Rope = proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = result = getCopyright(conf, cfile) + if conf.hcrOn: add(result, "#define NIM_HOT_CODE_RELOADING\L") addIntTypes(result, conf) +proc getSomeNameForModule(m: PSym): Rope = + assert m.kind == skModule + assert m.owner.kind == skPackage + if {sfSystemModule, sfMainModule} * m.flags == {}: + result = m.owner.name.s.mangle.rope + result.add "_" + result.add m.name.s.mangle + +proc getSomeInitName(m: BModule, suffix: string): Rope = + if not m.hcrOn: + result = getSomeNameForModule(m.module) + result.add suffix + +proc getInitName(m: BModule): Rope = + if sfMainModule in m.module.flags: + # generate constant name for main module, for "easy" debugging. + result = rope"NimMainModule" + else: + result = getSomeInitName(m, "Init000") + +proc getDatInitName(m: BModule): Rope = getSomeInitName(m, "DatInit000") +proc getHcrInitName(m: BModule): Rope = getSomeInitName(m, "HcrInit000") + +proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): Rope + proc genMainProc(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed - const + var preMainCode: Rope + if m.hcrOn: + proc loadLib(handle: string, name: string): Rope = + let prc = magicsys.getCompilerProc(m.g.graph, name) + assert prc != nil + let n = newStrNode(nkStrLit, prc.annex.path.strVal) + n.info = prc.annex.path.info + appcg(m, result, "\tif (!($1 = #nimLoadLibrary($2)))$N" & + "\t\t#nimLoadLibraryError($2);$N", + [handle.rope, genStringLiteral(m, n)]) + + add(preMainCode, loadLib("hcr_handle", "hcrGetProc")) + add(preMainCode, "\tvoid* rtl_handle;$N") + add(preMainCode, loadLib("rtl_handle", "nimGC_setStackBottom")) + add(preMainCode, hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr")) + add(preMainCode, "\tinner = PreMain;$N") + add(preMainCode, "\tinitStackBottomWith_actual((void *)&inner);$N") + add(preMainCode, "\t(*inner)();$N") + else: + add(preMainCode, "\tPreMain();$N") + + let + # not a big deal if we always compile these 3 global vars... makes the HCR code easier + PosixCmdLine = + "int cmdCount;$N" & + "char** cmdLine;$N" & + "char** gEnv;$N" + # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. - PreMainBody = + PreMainBody = "$N" & "void PreMainInner(void) {$N" & "$2" & "$3" & "}$N$N" & + PosixCmdLine & "void PreMain(void) {$N" & "\tvoid (*volatile inner)(void);$N" & "\tinner = PreMainInner;$N" & @@ -1133,7 +1246,7 @@ proc genMainProc(m: BModule) = "\tNimMain();$N" MainProcsWithResult = - MainProcs & "\treturn nim_program_result;$N" + MainProcs & ("\treturn " & (if m.hcrOn: "*" else: "") & "nim_program_result;$N") NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" & "$1" & @@ -1142,7 +1255,7 @@ proc genMainProc(m: BModule) = NimMainProc = "N_CDECL(void, NimMain)(void) {$N" & "\tvoid (*volatile inner)(void);$N" & - "\tPreMain();$N" & + $preMainCode & "\tinner = NimMainInner;$N" & "$2" & "\t(*inner)();$N" & @@ -1150,12 +1263,6 @@ proc genMainProc(m: BModule) = NimMainBody = NimMainInner & NimMainProc - PosixNimMain = - "int cmdCount;$N" & - "char** cmdLine;$N" & - "char** gEnv;$N" & - NimMainBody - PosixCMain = "int main(int argc, char** args, char** env) {$N" & "\tcmdLine = args;$N" & @@ -1227,10 +1334,10 @@ proc genMainProc(m: BModule) = nimMain = PosixNimDllMain otherMain = PosixCDllMain elif m.config.target.targetOS == osStandalone: - nimMain = PosixNimMain + nimMain = NimMainBody otherMain = StandaloneCMain else: - nimMain = PosixNimMain + nimMain = NimMainBody otherMain = PosixCMain if optEndb in m.config.options: for i in 0..<m.config.m.fileInfos.len: @@ -1253,28 +1360,53 @@ proc genMainProc(m: BModule) = appcg(m, m.s[cfsProcs], otherMain, []) if m.config.cppCustomNamespace.len > 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) -proc getSomeInitName(m: PSym, suffix: string): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle - result.add suffix - -proc getInitName(m: PSym): Rope = - if sfMainModule in m.flags: - # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" - else: - result = getSomeInitName(m, "Init000") - -proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") - - proc registerModuleToMain(g: BModuleList; m: BModule) = + let + init = m.getInitName + datInit = m.getDatInitName + + if m.hcrOn: + var hcr_module_meta = "$nN_LIB_PRIVATE const char* hcr_module_list[] = {$n" % [] + let systemModulePath = getModuleDllPath(m, g.modules[g.graph.config.m.systemFileIdx.int].module) + let mainModulePath = getModuleDllPath(m, m.module) + if sfMainModule in m.module.flags: + addf(hcr_module_meta, "\t$1,$n", [systemModulePath]) + g.graph.importDeps.withValue(FileIndex(m.module.position), deps): + for curr in deps[]: + addf(hcr_module_meta, "\t$1,$n", [getModuleDllPath(m, g.modules[curr.int].module)]) + addf(hcr_module_meta, "\t\"\"};$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", + [($sigHash(m.module)).rope]) + if sfMainModule in m.module.flags: + add(g.mainModProcs, hcr_module_meta) + addf(g.mainModProcs, "static void* hcr_handle;$N", []) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [init]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [datInit]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void*, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*));$N", [m.getHcrInitName]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void);$N", []) + addf(g.mainModInit, "\t$1();$N", [init]) + addf(g.otherModsInit, "\thcrInit((void**)hcr_module_list, $1, $2, $3, hcr_handle, nimGetProcAddr);$n", + [mainModulePath, systemModulePath, datInit]) + addf(g.mainDatInit, "\t$1(hcr_handle, nimGetProcAddr);$N", [m.getHcrInitName]) + addf(g.mainDatInit, "\thcrAddModule($1);\n", [mainModulePath]) + addf(g.mainDatInit, "\tHcrCreateTypeInfos();$N", []) + # nasty nasty hack to get the command line functionality working with HCR + # register the 2 variables on behalf of the os module which might not even + # be loaded (in which case it will get collected but that is not a problem) + let osModulePath = ($systemModulePath).replace("stdlib_system", "stdlib_os").rope + addf(g.mainDatInit, "\thcrAddModule($1);\n", [osModulePath]) + add(g.mainDatInit, "\tint* cmd_count;\n") + add(g.mainDatInit, "\tchar*** cmd_line;\n") + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdCount\", sizeof(cmd_count), NULL, (void**)&cmd_count);$N", [osModulePath]) + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdLine\", sizeof(cmd_line), NULL, (void**)&cmd_line);$N", [osModulePath]) + add(g.mainDatInit, "\t*cmd_count = cmdCount;\n") + add(g.mainDatInit, "\t*cmd_line = cmdLine;\n") + else: + add(m.s[cfsInitProc], hcr_module_meta) + return + if m.s[cfsDatInitProc].len > 0: - let datInit = m.module.getDatInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) addf(g.mainDatInit, "\t$1();$N", [datInit]) @@ -1287,7 +1419,6 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = add(g.mainDatInit, ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")) if m.s[cfsInitProc].len > 0: - let init = m.module.getInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.module.flags: @@ -1302,10 +1433,14 @@ proc genDatInitCode(m: BModule) = ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed - var moduleDatInitRequired = false + var moduleDatInitRequired = m.hcrOn + + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), getDatInitName(m)] - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % - [getDatInitName(m.module)] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) for i in cfsTypeInit1..cfsDynLibInit: if m.s[i].len != 0: @@ -1319,45 +1454,73 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: add(m.s[cfsDatInitProc], prc) +# Very similar to the contents of symInDynamicLib - basically only the +# things needed for the hot code reloading runtime procs to be loaded +proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): Rope = + let prc = magicsys.getCompilerProc(m.g.graph, sym) + assert prc != nil + fillProcLoc(m, prc.ast[namePos]) + + var extname = prefix & sym + var tmp = mangleDynLibProc(prc) + prc.loc.r = tmp + prc.typ.sym = nil + + if not containsOrIncl(m.declaredThings, prc.id): + addf(m.s[cfsVars], "static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t)]) + + result = "\t$1 = ($2) $3($4, $5);$n" % + [tmp, getTypeDesc(m, prc.typ), getProcFunc.rope, handle.rope, makeCString(prefix & sym)] + proc genInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed appcg(m, m.s[cfsForwardTypes], frameDefines, [rope("#")]) - var moduleInitRequired = false - let initname = getInitName(m.module) - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % [initname] + var moduleInitRequired = m.hcrOn + let initname = getInitName(m) + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) if m.typeNodes > 0: - appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", - [m.typeNodesName, rope(m.typeNodes)]) + if m.hcrOn: + appcg(m, m.s[cfsTypeInit1], "\t#TNimNode* $1;$N", [m.typeNodesName]) + appcg(m, m.s[cfsTypeInit1], "\thcrRegisterGlobal($3, \"$1_$2\", sizeof(TNimNode) * $2, NULL, (void**)&$1);$N", + [m.typeNodesName, rope(m.typeNodes), getModuleDllPath(m, m.module)]) + else: + appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", + [m.typeNodesName, rope(m.typeNodes)]) if m.nimTypes > 0: appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", [m.nimTypesName, rope(m.nimTypes)]) + if m.hcrOn: + addf(prc, "\tint* nim_hcr_dummy_ = 0;$n" & + "\tNIM_BOOL nim_hcr_do_init_ = " & + "hcrRegisterGlobal($1, \"module_initialized_\", 1, NULL, (void**)&nim_hcr_dummy_);$n", + [getModuleDllPath(m, m.module)]) + + template writeSection(thing: untyped, section: TCProcSection, addHcrGuards = false) = + if m.thing.s(section).len > 0: + moduleInitRequired = true + if addHcrGuards: add(prc, "\tif (nim_hcr_do_init_) {\n\n") + add(prc, genSectionStart(section, m.config)) + add(prc, m.thing.s(section)) + add(prc, genSectionEnd(section, m.config)) + if addHcrGuards: add(prc, "\n\t} // nim_hcr_do_init_\n") + if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: # Give this small function its own scope addf(prc, "{$N", []) # Keep a bogus frame in case the code needs one add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - if m.preInitProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.preInitProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) - - if m.preInitProc.s(cpsInit).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.preInitProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - if m.preInitProc.s(cpsStmts).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.preInitProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(preInitProc, cpsLocals) + writeSection(preInitProc, cpsInit, m.hcrOn) + writeSection(preInitProc, cpsStmts) addf(prc, "}$N", []) # add new scope for following code, because old vcc compiler need variable @@ -1367,11 +1530,7 @@ proc genInitCode(m: BModule) = moduleInitRequired = true add(prc, initGCFrame(m.initProc)) - if m.initProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.initProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) + writeSection(initProc, cpsLocals) if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: moduleInitRequired = true @@ -1385,13 +1544,8 @@ proc genInitCode(m: BModule) = else: add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.initProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.initProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(initProc, cpsInit, m.hcrOn) + writeSection(initProc, cpsStmts) if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: add(prc, deinitFrame(m.initProc)) @@ -1407,6 +1561,19 @@ proc genInitCode(m: BModule) = # that would lead to a *nesting* of merge sections which the merger does # not support. So we add it to another special section: ``cfsInitProc`` + if m.hcrOn: + var procsToLoad = @["hcrRegisterProc", "hcrGetProc", "hcrRegisterGlobal", "hcrGetGlobal"] + + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, $1)(void* handle, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*)) {$N", [getHcrInitName(m)]) + if sfMainModule in m.module.flags: + # additional procs to load + procsToLoad.add("hcrInit") + procsToLoad.add("hcrAddModule") + # load procs + for curr in procsToLoad: + add(m.s[cfsInitProc], hcrGetProcLoadCode(m, curr, "", "handle", "getProcAddr")) + addf(m.s[cfsInitProc], "}$N$N", []) + for i, el in pairs(m.extensionLoaders): if el != nil: let ex = "NIM_EXTERNC N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" % @@ -1418,6 +1585,12 @@ proc genInitCode(m: BModule) = add(m.s[cfsInitProc], prc) genDatInitCode(m) + + if m.hcrOn: + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void) {$N", []) + add(m.s[cfsInitProc], m.hcrCreateTypeInfosProc) + addf(m.s[cfsInitProc], "}$N$N", []) + registerModuleToMain(m.g, m) proc genModule(m: BModule, cfile: Cfile): Rope = @@ -1445,7 +1618,7 @@ proc genModule(m: BModule, cfile: Cfile): Rope = if m.s[cfsInitProc].len > 0: moduleIsEmpty = false add(result, m.s[cfsInitProc]) - if m.s[cfsDatInitProc].len > 0: + if m.s[cfsDatInitProc].len > 0 or m.hcrOn: moduleIsEmpty = false add(result, m.s[cfsDatInitProc]) @@ -1565,6 +1738,25 @@ when false: readMergeInfo(getCFile(m), m) result = m +proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, inInitGuard) + else: + let stmtShouldExecute = n.kind in {nkVarSection, nkLetSection} or + nfExecuteOnReload in n.flags + if inInitGuard: + if stmtShouldExecute: + endBlock(p) + inInitGuard = false + else: + if not stmtShouldExecute: + line(p, cpsStmts, "if (nim_hcr_do_init_)\n") + startBlock(p) + inInitGuard = true + + genStmts(p, n) + proc myProcess(b: PPassContext, n: PNode): PNode = result = n if b == nil: return @@ -1573,8 +1765,11 @@ proc myProcess(b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! - let tranformed_n = transformStmt(m.g.graph, m.module, n) - genStmts(m.initProc, tranformed_n) + let transformed_n = transformStmt(m.g.graph, m.module, n) + if m.hcrOn: + addHcrInitGuards(m.initProc, transformed_n, m.inHcrInitGuard) + else: + genStmts(m.initProc, transformed_n) proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = result = true @@ -1674,7 +1869,24 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) genStmts(m.initProc, n) + if m.hcrOn: + # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init) + discard cgsym(m, "programResult") + if m.inHcrInitGuard: + endBlock(m.initProc) + if sfMainModule in m.module.flags: + if m.hcrOn: + # pull ("define" since they are inline when HCR is on) these functions in the main file + # so it can load the HCR runtime and later pass the library handle to the HCR runtime which + # will in turn pass it to the other modules it initializes so they can initialize the + # register/get procs so they don't have to have the definitions of these functions as well + discard cgsym(m, "nimLoadLibrary") + discard cgsym(m, "nimLoadLibraryError") + discard cgsym(m, "nimGetProcAddr") + discard cgsym(m, "procAddrError") + discard cgsym(m, "rawWrite") + # raise dependencies on behalf of genMainProc if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: discard cgsym(m, "initStackBottomWith") @@ -1709,14 +1921,11 @@ proc genForwardedProcs(g: BModuleList) = proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = let g = BModuleList(backend) + g.config = config + # we need to process the transitive closure because recursive module # deps are allowed (and the system module is processed in the wrong # order anyway) - g.config = config - let (outDir, _, _) = splitFile(config.outfile) - if not outDir.isEmpty: - createDir(outDir) - genForwardedProcs(g) for m in cgenModules(g): diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 50b484e31..7b1e7c123 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -98,6 +98,7 @@ type TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] + TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]] Codegenflag* = enum preventStackTrace, # true if stack traces need to be prevented @@ -118,7 +119,7 @@ type generatedHeader*: BModule breakPointId*: int breakpoints*: Rope # later the breakpoints are inserted into the main proc - typeInfoMarker*: TypeCache + typeInfoMarker*: TypeCacheWithOwner config*: ConfigRef graph*: ModuleGraph strVersion*, seqVersion*: int # version of the string/seq implementation to use @@ -150,6 +151,8 @@ type typeInfoMarker*: TypeCache # needed for generating type information initProc*: BProc # code for init procedure preInitProc*: BProc # code executed before the init proc + hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on + inHcrInitGuard*: bool # We are currently withing a HCR reloading guard. typeStack*: TTypeSeq # used for type generation dataCache*: TNodeTable typeNodes*, nimTypes*: int # used for type info generation @@ -189,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc = result.sigConflicts = initCountTable[string]() proc newModuleList*(g: ModuleGraph): BModuleList = - BModuleList(typeInfoMarker: initTable[SigHash, Rope](), config: g.config, - graph: g, nimtvDeclared: initIntSet()) + BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](), + config: g.config, graph: g, nimtvDeclared: initIntSet()) iterator cgenModules*(g: BModuleList): BModule = for m in g.modules_closed: diff --git a/compiler/commands.nim b/compiler/commands.nim index 56fe8f205..1f46abca4 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -387,7 +387,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.nimcacheDir = processPath(conf, arg, info, true) of "out", "o": expectArg(conf, switch, arg, pass, info) - conf.outFile = AbsoluteFile arg + let f = splitFile(arg.expandTilde) + conf.outFile = RelativeFile f.name + conf.outDir = toAbsoluteDir f.dir + of "outdir": + expectArg(conf, switch, arg, pass, info) + conf.outDir = toAbsoluteDir arg.expandTilde of "docseesrcurl": expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg @@ -487,9 +492,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if optMemTracker in conf.options: defineSymbol(conf.symbols, "memtracker") else: undefSymbol(conf.symbols, "memtracker") of "hotcodereloading": - processOnOffSwitch(conf, {optHotCodeReloading}, arg, pass, info) - if optHotCodeReloading in conf.options: defineSymbol(conf.symbols, "hotcodereloading") - else: undefSymbol(conf.symbols, "hotcodereloading") + processOnOffSwitchG(conf, {optHotCodeReloading}, arg, pass, info) + if conf.hcrOn: + defineSymbol(conf.symbols, "hotcodereloading") + defineSymbol(conf.symbols, "useNimRtl") + else: + undefSymbol(conf.symbols, "hotcodereloading") + undefSymbol(conf.symbols, "useNimRtl") of "oldnewlines": case arg.normalize of "","on": @@ -787,7 +796,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: # support UNIX style filenames everywhere for portable build scripts: - config.projectName = unixToNativePath(p.key) + if config.projectName.len == 0: + config.projectName = unixToNativePath(p.key) config.arguments = cmdLineRest(p) result = true inc argsCount diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 5e7ce3a08..57dd55132 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -82,7 +82,9 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasTypeof") defineSymbol("nimErrorProcCanHaveBody") defineSymbol("nimHasInstantiationOfInMacro") + defineSymbol("nimHasHotCodeReloading") defineSymbol("nimHasNilSeqs") + defineSymbol("nimHasSignatureHashInMacro") for f in low(Feature)..high(Feature): defineSymbol("nimHas" & $f) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5af4c464e..28d53533f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -100,14 +100,12 @@ proc parseRst(text, filename: string, proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile = if optWholeProject in conf.globalOptions: - # This is correct, for 'nim doc --project' we interpret the '--out' option as an - # absolute directory, not as a filename! - let d = if conf.outFile.isEmpty: conf.projectPath / dir else: AbsoluteDir(conf.outFile) + let d = if conf.outDir.isEmpty: conf.projectPath / dir else: conf.outDir createDir(d) result = d / changeFileExt(filename, ext) elif guessTarget: - let d = if not conf.outFile.isEmpty: splitFile(conf.outFile).dir - else: conf.projectPath + let d = if not conf.outDir.isEmpty: conf.outDir + else: conf.projectPath createDir(d) result = d / changeFileExt(filename, ext) else: @@ -952,9 +950,8 @@ proc genOutFile(d: PDoc): Rope = proc generateIndex*(d: PDoc) = if optGenIndex in d.conf.globalOptions: - let dir = if d.conf.outFile.isEmpty: d.conf.projectPath / RelativeDir"htmldocs" - elif optWholeProject in d.conf.globalOptions: AbsoluteDir(d.conf.outFile) - else: AbsoluteDir(d.conf.outFile.string.splitFile.dir) + let dir = if not d.conf.outDir.isEmpty: d.conf.outDir + else: d.conf.projectPath / RelativeDir"htmldocs" createDir(dir) let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename, d.conf.projectPath), IndexExt) @@ -995,6 +992,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) = "\" for writing") proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index f2ac6c1a9..2232601f4 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -29,7 +29,6 @@ template shouldProcess(g): bool = template closeImpl(body: untyped) {.dirty.} = var g = PGen(p) let useWarning = sfMainModule notin g.module.flags - #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId if shouldProcess(g): body try: diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2c5af6433..18a81ca96 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - lineinfos, std / sha1, streams, pathutils + lineinfos, std / sha1, streams, pathutils, sequtils, times type TInfoCCProp* = enum # properties of the C compiler: @@ -468,8 +468,8 @@ proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % cmd) -proc generateScript(conf: ConfigRef; projectFile: AbsoluteFile, script: Rope) = - let (_, name, _) = splitFile(projectFile) +proc generateScript(conf: ConfigRef; script: Rope) = + let (_, name, _) = splitFile(conf.outFile.string) let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name, platform.OS[conf.target.targetOS].scriptExt)) if writeRope(script, filename): @@ -551,14 +551,14 @@ proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler else: getCompilerExe(conf, compiler, AbsoluteFile"") -proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile): string = +proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string = var c = conf.cCompiler var options = cFileSpecificOptions(conf, cfile.cname) var exe = getConfigVar(conf, c, ".exe") if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) if needsExeExt(conf): exe = addFileExt(exe, "exe") - if optGenDynLib in conf.globalOptions and + if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and ospNeedsPIC in platform.OS[conf.target.targetOS].props: add(options, ' ' & CC[c].pic) @@ -648,12 +648,14 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) -proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, +proc compileCFiles(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, prettyCmds: var TStringSeq) = + var currIdx = 0 for it in list: # call the C compiler for the .c file: if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(conf, it) + var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1) + inc currIdx if optCompileOnly notin conf.globalOptions: add(cmds, compileCmd) let (_, name, _) = splitFile(it.cname) @@ -662,7 +664,8 @@ proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var add(script, compileCmd) add(script, "\n") -proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): string = +proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, + objfiles: string, isDllBuild: bool): string = if optGenStaticLib in conf.globalOptions: var libname: string if not conf.outFile.isEmpty: @@ -672,7 +675,7 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s else: libname = (libNameTmpl(conf) % splitFile(conf.projectName).name) result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname), - "objfiles", objfiles] + "objfiles", objfiles] else: var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe") if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler) @@ -684,28 +687,16 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s CC[conf.cCompiler].buildGui else: "" - var exefile, builddll: string - if optGenDynLib in conf.globalOptions: - exefile = platform.OS[conf.target.targetOS].dllFrmt % splitFile(projectfile).name - builddll = CC[conf.cCompiler].buildDll - else: - exefile = splitFile(projectfile).name & platform.OS[conf.target.targetOS].exeExt - builddll = "" - if not conf.outFile.isEmpty: - exefile = conf.outFile.string.expandTilde - if not exefile.isAbsolute(): - exefile = getCurrentDir() / exefile - if not noAbsolutePaths(conf): - if not exefile.isAbsolute(): - exefile = string(splitFile(projectfile).dir / RelativeFile(exefile)) + let builddll = if isDllBuild: CC[conf.cCompiler].buildDll else: "" + let exefile = quoteShell(output) + when false: if optCDebug in conf.globalOptions: writeDebugInfo(exefile.changeFileExt("ndb")) - exefile = quoteShell(exefile) # Map files are required by Nintendo Switch compilation. They are a list # of all function calls in the library and where they come from. - let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(projectFile).name & ".map")) + let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(output).name & ".map")) let linkOptions = getLinkOptions(conf) & " " & getConfigVar(conf, conf.cCompiler, ".options.linker") @@ -723,6 +714,46 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s "objfiles", objfiles, "exefile", exefile, "nim", quoteShell(getPrefixDir(conf)), "lib", quoteShell(conf.libpath)]) + # On windows the debug information for binaries is emitted in a separate .pdb + # file and the binaries (.dll and .exe) contain a full path to that .pdb file. + # This is a problem for hot code reloading because even when we copy the .dll + # and load the copy so the build process may overwrite the original .dll on + # the disk (windows locks the files of running binaries) the copy still points + # to the original .pdb (and a simple copy of the .pdb won't help). This is a + # problem when a debugger is attached to the program we are hot-reloading. + # This problem is nonexistent on Unix since there by default debug symbols + # are embedded in the binaries so loading a copy of a .so will be fine. There + # is the '/Z7' flag for the MSVC compiler to embed the debug info of source + # files into their respective .obj files but the linker still produces a .pdb + # when a final .dll or .exe is linked so the debug info isn't embedded. + # There is also the issue that even when a .dll is unloaded the debugger + # still keeps the .pdb for that .dll locked. This is a major problem and + # because of this we cannot just alternate between 2 names for a .pdb file + # when rebuilding a .dll - instead we need to accumulate differently named + # .pdb files in the nimcache folder - this is the easiest and most reliable + # way of being able to debug and rebuild the program at the same time. This + # is accomplished using the /PDB:<filename> flag (there also exists the + # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are + # atleast 5-10mb big and will quickly accumulate. There is a hacky solution: + # we could try to delete all .pdb files with a pattern and swallow exceptions. + # + # links about .pdb files and hot code reloading: + # https://ourmachinery.com/post/dll-hot-reloading-in-theory-and-practice/ + # https://ourmachinery.com/post/little-machines-working-together-part-2/ + # https://github.com/fungos/cr + # https://fungos.github.io/blog/2017/11/20/cr.h-a-simple-c-hot-reload-header-only-library/ + # on forcing the debugger to unlock a locked .pdb of an unloaded library: + # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/ + # and a bit about the .pdb format in case that is ever needed: + # https://github.com/crosire/blink + # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles + if conf.hcrOn and conf.cCompiler == ccVcc: + let t = now() + let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb" + result.add " /link /PDB:" & pdb + +template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string = + getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions) template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped): typed = try: @@ -799,7 +830,17 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = finally: removeFile(linkerArgs) -proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc getObjFilePath(conf: ConfigRef, f: CFile): string = + if noAbsolutePaths(conf): f.obj.extractFilename + else: f.obj.string + +proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): AbsoluteFile = + let basename = splitFile(objFile).name + let targetName = if isMain: basename & ".exe" + else: platform.OS[conf.target.targetOS].dllFrmt % basename + result = conf.nimcacheDir / RelativeFile(targetName) + +proc callCCompiler*(conf: ConfigRef) = var linkCmd: string if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: @@ -813,7 +854,7 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = when declared(echo): let cmd = prettyCmds[idx] if cmd != "": echo cmd - compileCFile(conf, conf.toCompile, script, cmds, prettyCmds) + compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) if optNoLinking notin conf.globalOptions: @@ -824,31 +865,64 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = add(objfiles, ' ') add(objfiles, quoteShell( addFileExt(objFile, CC[conf.cCompiler].objExt))) - for x in conf.toCompile: - let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string - add(objfiles, ' ') - add(objfiles, quoteShell(objFile)) - linkCmd = getLinkCmd(conf, projectfile, objfiles) - if optCompileOnly notin conf.globalOptions: - if defined(windows) and linkCmd.len > 8_000: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) + if conf.hcrOn: # lets assume that optCompileOnly isn't on + cmds = @[] + let mainFileIdx = conf.toCompile.len - 1 + for idx, x in conf.toCompile: + # don't relink each of the many binaries (one for each source file) if the nim code is + # cached because that would take too much time for small changes - the only downside to + # this is that if an external-to-link file changes the final target wouldn't be relinked + if x.flags.contains(CfileFlag.Cached): continue + # we pass each object file as if it is the project file - a .dll will be created for each such + # object file in the nimcache directory, and only in the case of the main project file will + # there be probably an executable (if the project is such) which will be copied out of the nimcache + let objFile = conf.getObjFilePath(x) + let buildDll = idx != mainFileIdx + let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll) + add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll)) + # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache + # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used + if conf.cCompiler == ccVcc: + for pdb in walkFiles(objFile & ".*.pdb"): + discard tryRemoveFile(pdb) + # execute link commands in parallel - output will be a bit different + # if it fails than that from execLinkCmd() but that doesn't matter + prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) + execCmdsInParallel(conf, cmds, prettyCb) + # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination + if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached): + let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) + var src = conf.hcrLinkTargetName(mainObjFile, true) + var dst = conf.prepareToWriteOutput + copyFileWithPermissions(src.string, dst.string) + else: + for x in conf.toCompile: + let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string + add(objfiles, ' ') + add(objfiles, quoteShell(objFile)) + let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput + else: AbsoluteFile(conf.projectName) + linkCmd = getLinkCmd(conf, mainOutput, objfiles) + if optCompileOnly notin conf.globalOptions: + if defined(windows) and linkCmd.len > 8_000: + # Windows's command line limit is about 8K (don't laugh...) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) else: linkCmd = "" if optGenScript in conf.globalOptions: add(script, linkCmd) add(script, "\n") - generateScript(conf, projectfile, script) + generateScript(conf, script) #from json import escapeJson import json -proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc writeJsonBuildInstructions*(conf: ConfigRef) = template lit(x: untyped) = f.write x template str(x: untyped) = when compiles(escapeJson(x, buf)): @@ -895,7 +969,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = var buf = newStringOfCap(50) - let jsonFile = toGeneratedFile(conf, projectfile, "json") + let jsonFile = conf.nimcacheDir / RelativeFile(conf.projectName & ".json") var f: File if open(f, jsonFile.string, fmWrite): @@ -907,7 +981,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) lit "],\L\"linkcmd\": " - str getLinkCmd(conf, projectfile, objfiles) + str getLinkCmd(conf, conf.absOutFile, objfiles) lit "\L}\L" close(f) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index cad8fc990..e3ca6830c 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -83,7 +83,6 @@ type forwarded: seq[PSym] generatedSyms: IntSet typeInfoGenerated: IntSet - classes: seq[(PType, Rope)] unique: int # for temp identifier generation PProc = ref TProc @@ -133,7 +132,6 @@ proc newGlobals(): PGlobals = result.forwarded = @[] result.generatedSyms = initIntSet() result.typeInfoGenerated = initIntSet() - result.classes = @[] proc initCompRes(r: var TCompRes) = r.address = nil @@ -250,7 +248,7 @@ proc mangleName(m: BModule, s: PSym): Rope = result = rope(x) # From ES5 on reserved words can be used as object field names if s.kind != skField: - if optHotCodeReloading in m.config.options: + if m.config.hcrOn: # When hot reloading is enabled, we must ensure that the names # of functions and types will be preserved across rebuilds: add(result, idOrSig(s, m.module.name.s, m.sigConflicts)) @@ -1206,19 +1204,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = gen(p, n.sons[0].sons[0], r) else: internalError(p.config, n.sons[0].info, "genAddr: " & $n.sons[0].kind) -proc thisParam(p: PProc; typ: PType): PType = - discard - proc attachProc(p: PProc; content: Rope; s: PSym) = - let otyp = thisParam(p, s.typ) - if otyp != nil: - for i, cls in p.g.classes: - if sameType(cls[0], otyp): - add(p.g.classes[i][1], content) - return - p.g.classes.add((otyp, content)) - else: - add(p.g.code, content) + add(p.g.code, content) proc attachProc(p: PProc; s: PSym) = let newp = genProc(p, s) @@ -1460,9 +1447,6 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) = genArgs(p, n, r, 2) proc genCall(p: PProc, n: PNode, r: var TCompRes) = - if n.sons[0].kind == nkSym and thisParam(p, n.sons[0].typ) != nil: - genInfixCall(p, n, r) - return gen(p, n.sons[0], r) genArgs(p, n, r) if n.typ != nil: @@ -1606,13 +1590,14 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = s: Rope varCode: string varName = mangleName(p.module, v) - useReloadingGuard = sfGlobal in v.flags and optHotCodeReloading in p.config.options + useReloadingGuard = sfGlobal in v.flags and p.config.hcrOn if v.constraint.isNil: if useReloadingGuard: lineF(p, "var $1;$n", varName) lineF(p, "if ($1 === undefined) {$n", varName) varCode = $varName + inc p.extraIndent else: varCode = "var $2" else: @@ -1664,6 +1649,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, s]) if useReloadingGuard: + dec p.extraIndent lineF(p, "}$n") proc genVarStmt(p: PProc, n: PNode) = @@ -2204,7 +2190,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = else: result = ~"\L" - if optHotCodeReloading in p.config.options: + if p.config.hcrOn: # Here, we introduce thunks that create the equivalent of a jump table # for all global functions, because references to them may be stored # in JavaScript variables. The added indirection ensures that such @@ -2440,13 +2426,52 @@ proc genHeader(): Rope = "if (typeof Float64Array === 'undefined') Float64Array = Array;$n") % [rope(VersionAsString)] +proc addHcrInitGuards(p: PProc, n: PNode, + moduleLoadedVar: Rope, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, moduleLoadedVar, inInitGuard) + else: + let stmtShouldExecute = n.kind in { + nkProcDef, nkFuncDef, nkMethodDef,nkConverterDef, + nkVarSection, nkLetSection} or nfExecuteOnReload in n.flags + + if inInitGuard: + if stmtShouldExecute: + dec p.extraIndent + line(p, "}\L") + inInitGuard = false + else: + if not stmtShouldExecute: + lineF(p, "if ($1 == undefined) {$n", [moduleLoadedVar]) + inc p.extraIndent + inInitGuard = true + + genStmt(p, n) + proc genModule(p: PProc, n: PNode) = if optStackTrace in p.options: add(p.body, frameCreate(p, makeJSString("module " & p.module.module.name.s), makeJSString(toFilename(p.config, p.module.module.info)))) let n_transformed = transformStmt(p.module.graph, p.module.module, n) - genStmt(p, n_transformed) + if p.config.hcrOn and n.kind == nkStmtList: + let moduleSym = p.module.module + var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" & + idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts) + lineF(p, "var $1;$n", [moduleLoadedVar]) + var inGuardedBlock = false + + addHcrInitGuards(p, n_transformed, moduleLoadedVar, inGuardedBlock) + + if inGuardedBlock: + dec p.extraIndent + line(p, "}\L") + + lineF(p, "$1 = true;$n", [moduleLoadedVar]) + else: + genStmt(p, n_transformed) + if optStackTrace in p.options: add(p.body, frameDestroy(p)) @@ -2487,44 +2512,14 @@ proc getClassName(t: PType): Rope = if s.loc.r != nil: result = s.loc.r else: result = rope(s.name.s) -proc genClass(conf: ConfigRef; obj: PType; content: Rope; ext: string) = - let cls = getClassName(obj) - let t = skipTypes(obj, abstractPtrs) - let extends = if t.kind == tyObject and t.sons[0] != nil: - " extends " & getClassName(t.sons[0]) - else: nil - let result = ("<?php$n" & - "/* Generated by the Nim Compiler v$# */$n" & - "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & - "require_once \"nimsystem.php\";$n" & - "class $#$# {$n$#$n}$n") % - [rope(VersionAsString), cls, extends, content] - - let outfile = changeFileExt(completeCFilePath(conf, AbsoluteFile($cls)), ext) - discard writeRopeIfNotEqual(result, outfile) - proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = result = myProcess(b, n) var m = BModule(b) if passes.skipCodegen(m.config, n): return n if sfMainModule in m.module.flags: - let globals = PGlobals(graph.backend) - let ext = "js" - let f = if globals.classes.len == 0: toFilename(m.config, FileIndex m.module.position) - else: "nimsystem" let code = wholeCode(graph, m) - let outfile = - if not m.config.outFile.isEmpty: - if m.config.outFile.string.isAbsolute: m.config.outFile - else: AbsoluteFile(getCurrentDir() / m.config.outFile.string) - else: - changeFileExt(completeCFilePath(m.config, AbsoluteFile f), ext) - let (outDir, _, _) = splitFile(outfile) - if not outDir.isEmpty: - createDir(outDir) - discard writeRopeIfNotEqual(genHeader() & code, outfile) - for obj, content in items(globals.classes): - genClass(m.config, obj, content, ext) + let outFile = m.config.prepareToWriteOutput() + discard writeRopeIfNotEqual(genHeader() & code, outFile) proc myOpen(graph: ModuleGraph; s: PSym): PPassContext = result = newModule(graph, s) diff --git a/compiler/layouter.nim b/compiler/layouter.nim index 8605ade45..7b91b7bc4 100644 --- a/compiler/layouter.nim +++ b/compiler/layouter.nim @@ -56,12 +56,13 @@ proc openEmitter*(em: var Emitter, cache: IdentCache; em.lastLineNumber = 1 proc closeEmitter*(em: var Emitter) = - if fileExists(em.config.outFile) and readFile(em.config.outFile.string) == em.content: + let outFile = em.config.absOutFile + if fileExists(outFile) and readFile(outFile.string) == em.content: discard "do nothing, see #9499" return - var f = llStreamOpen(em.config.outFile, fmWrite) + var f = llStreamOpen(outFile, fmWrite) if f == nil: - rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile.string) + rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string) return f.llStreamWrite em.content llStreamClose(f) diff --git a/compiler/main.nim b/compiler/main.nim index c1477a22b..282f7d814 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -33,8 +33,9 @@ proc semanticPasses(g: ModuleGraph) = registerPass g, verbosePass registerPass g, semPass -proc writeDepsFile(g: ModuleGraph; project: AbsoluteFile) = - let f = open(changeFileExt(project, "deps").string, fmWrite) +proc writeDepsFile(g: ModuleGraph) = + let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps") + let f = open(fname.string, fmWrite) for m in g.modules: if m != nil: f.writeLine(toFullPath(g.config, m.position.FileIndex)) @@ -48,7 +49,7 @@ proc commandGenDepend(graph: ModuleGraph) = registerPass(graph, gendependPass) compileProject(graph) let project = graph.config.projectFull - writeDepsFile(graph, project) + writeDepsFile(graph) generateDot(graph, project) execExternalProgram(graph.config, "dot -Tpng -o" & changeFileExt(project, "png").string & @@ -62,6 +63,7 @@ proc commandCheck(graph: ModuleGraph) = when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; json: bool) = + graph.config.outDir = AbsoluteDir(graph.config.outDir / graph.config.outFile) graph.config.errorMax = high(int) # do not stop after first error semanticPasses(graph) if json: registerPass(graph, docgen2JsonPass) @@ -71,6 +73,16 @@ when not defined(leanCompiler): proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config + + if conf.outDir.isEmpty: + conf.outDir = conf.projectPath + if conf.outFile.isEmpty: + let targetName = if optGenDynLib in conf.globalOptions: + platform.OS[conf.target.targetOS].dllFrmt % conf.projectName + else: + conf.projectName & platform.OS[conf.target.targetOS].exeExt + conf.outFile = RelativeFile targetName + extccomp.initVars(conf) semanticPasses(graph) registerPass(graph, cgenPass) @@ -80,11 +92,12 @@ proc commandCompileToC(graph: ModuleGraph) = return # issue #9933 cgenWriteModules(graph.backend, conf) if conf.cmd != cmdRun: - let proj = changeFileExt(conf.projectFull, "") - extccomp.callCCompiler(conf, proj) - extccomp.writeJsonBuildInstructions(conf, proj) + extccomp.callCCompiler(conf) + # for now we do not support writing out a .json file with the build instructions when HCR is on + if not conf.hcrOn: + extccomp.writeJsonBuildInstructions(conf) if optGenScript in graph.config.globalOptions: - writeDepsFile(graph, toGeneratedFile(conf, proj, "")) + writeDepsFile(graph) proc commandJsonScript(graph: ModuleGraph) = let proj = changeFileExt(graph.config.projectFull, "") @@ -93,6 +106,12 @@ proc commandJsonScript(graph: ModuleGraph) = when not defined(leanCompiler): proc commandCompileToJS(graph: ModuleGraph) = let conf = graph.config + + if conf.outDir.isEmpty: + conf.outDir = conf.projectPath + if conf.outFile.isEmpty: + conf.outFile = RelativeFile(conf.projectName & ".js") + #incl(gGlobalOptions, optSafeCode) setTarget(graph.config.target, osJS, cpuJS) #initDefines() @@ -102,7 +121,7 @@ when not defined(leanCompiler): registerPass(graph, JSgenPass) compileProject(graph) if optGenScript in graph.config.globalOptions: - writeDepsFile(graph, toGeneratedFile(conf, conf.projectFull, "")) + writeDepsFile(graph) proc interactivePasses(graph: ModuleGraph) = initDefines(graph.config.symbols) @@ -193,6 +212,12 @@ proc mainCommand*(graph: ModuleGraph) = quit "compiler wasn't built with JS code generator" else: conf.cmd = cmdCompileToJS + if conf.hcrOn: + # XXX: At the moment, system.nim cannot be compiled in JS mode + # with "-d:useNimRtl". The HCR option has been processed earlier + # and it has added this define implictly, so we must undo that here. + # A better solution might be to fix system.nim + undefSymbol(conf.symbols, "useNimRtl") commandCompileToJS(graph) of "doc0": when defined(leanCompiler): @@ -287,6 +312,7 @@ proc mainCommand*(graph: ModuleGraph) = (key: "project_path", val: %conf.projectFull.string), (key: "defined_symbols", val: definedSymbols), (key: "lib_paths", val: %libpaths), + (key: "outdir", val: %conf.outDir.string), (key: "out", val: %conf.outFile.string), (key: "nimcache", val: %getNimcacheDir(conf).string), (key: "hints", val: hints), diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 63fa597ff..8c081da08 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -33,6 +33,7 @@ type modules*: seq[PSym] ## indexed by int32 fileIdx packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. + importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. invalidTransitiveClosure: bool inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the @@ -118,6 +119,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result = ModuleGraph() initStrTable(result.packageSyms) result.deps = initIntSet() + result.importDeps = initTable[FileIndex, seq[FileIndex]]() result.modules = @[] result.importStack = @[] result.inclToMod = initTable[FileIndex, FileIndex]() diff --git a/compiler/modules.nim b/compiler/modules.nim index 442305a06..3d8ced35d 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -12,7 +12,7 @@ import ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod, - lineinfos, pathutils + lineinfos, pathutils, tables proc resetSystemArtifacts*(g: ModuleGraph) = magicsys.resetSysTypes(g) @@ -91,6 +91,9 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc assert graph.config != nil result = compileModule(graph, fileIdx, {}) graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) #if sfSystemModule in result.flags: # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: diff --git a/compiler/nim.nim b/compiler/nim.nim index cbd9d6f39..5ca650789 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -21,9 +21,11 @@ when defined(i386) and defined(windows) and defined(vcc): import commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, extccomp, strutils, os, osproc, platform, main, parseopt, - nodejs, scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper, + scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper, pathutils +include nodejs + when hasTinyCBackend: import tccgen @@ -78,23 +80,10 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if conf.cmd == cmdRun: tccgen.run(conf.arguments) if optRun in conf.globalOptions: + var ex = quoteShell conf.absOutFile if conf.cmd == cmdCompileToJS: - var ex: string - if not conf.outFile.isEmpty: - ex = conf.outFile.prependCurDir.quoteShell - else: - ex = quoteShell( - completeCFilePath(conf, changeFileExt(conf.projectFull, "js").prependCurDir)) execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments) else: - var binPath: AbsoluteFile - if not conf.outFile.isEmpty: - # If the user specified an outFile path, use that directly. - binPath = conf.outFile.prependCurDir - else: - # Figure out ourselves a valid binary name. - binPath = changeFileExt(conf.projectFull, ExeExt).prependCurDir - var ex = quoteShell(binPath) execExternalProgram(conf, ex & ' ' & conf.arguments) when declared(GC_setMaxPause): diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 7f9f28aaf..eaf99729d 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -6,3 +6,4 @@ proc findNodeJs*(): string = result = findExe("node") if result == "": result = findExe("iojs") + diff --git a/compiler/options.nim b/compiler/options.nim index c9f884986..037febc28 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -38,7 +38,6 @@ type # please make sure we have under 32 options # evaluation optPatterns, # en/disable pattern matching optMemTracker, - optHotCodeReloading, optLaxStrings, optNilSeqs, optOldAst @@ -81,6 +80,7 @@ type # please make sure we have under 32 options optMixedMode # true if some module triggered C++ codegen optListFullPaths # use full paths in toMsgFilename, toFilename optNoNimblePath + optHotCodeReloading optDynlibOverrideAll TGlobalOptions* = set[TGlobalOption] @@ -215,7 +215,8 @@ type packageCache*: StringTableRef searchPaths*: seq[AbsoluteDir] lazyPaths*: seq[AbsoluteDir] - outFile*: AbsoluteFile + outFile*: RelativeFile + outDir*: AbsoluteDir prefixDir*, libpath*, nimcacheDir*: AbsoluteDir dllOverrides, moduleOverrides*: StringTableRef projectName*: string # holds a name like 'nim' @@ -253,6 +254,8 @@ type severity: Severity) {.closure.} cppCustomNamespace*: string +proc hcrOn*(conf: ConfigRef): bool = return optHotCodeReloading in conf.globalOptions + template depConfigFields*(fn) {.dirty.} = fn(target) fn(options) @@ -314,7 +317,9 @@ proc newConfigRef*(): ConfigRef = packageCache: newPackageCache(), searchPaths: @[], lazyPaths: @[], - outFile: AbsoluteFile"", prefixDir: AbsoluteDir"", + outFile: RelativeFile"", + outDir: AbsoluteDir"", + prefixDir: AbsoluteDir"", libpath: AbsoluteDir"", nimcacheDir: AbsoluteDir"", dllOverrides: newStringTable(modeCaseInsensitive), moduleOverrides: newStringTable(modeStyleInsensitive), @@ -430,9 +435,6 @@ const const oKeepVariableNames* = true -template compilingLib*(conf: ConfigRef): bool = - gGlobalOptions * {optGenGuiApp, optGenDynLib} != {} - proc mainCommandArg*(conf: ConfigRef): string = ## This is intended for commands like check or parse ## which will work on the main project file unless @@ -452,8 +454,16 @@ proc setConfigVar*(conf: ConfigRef; key, val: string) = conf.configVars[key] = val proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile = - if not conf.outFile.isEmpty: result = conf.outFile - else: result = conf.projectPath / changeFileExt(filename, ext) + result = (if conf.outFile.isEmpty: conf.projectPath else: conf.outDir) / + changeFileExt(filename, ext) + +proc absOutFile*(conf: ConfigRef): AbsoluteFile = + conf.outDir / conf.outFile + +proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile = + ## Create the output directory and returns a full path to the output file + createDir conf.outDir + return conf.outDir / conf.outFile proc getPrefixDir*(conf: ConfigRef): AbsoluteDir = ## Gets the prefix dir, usually the parent directory where the binary resides. diff --git a/compiler/passes.nim b/compiler/passes.nim index f45ee03f7..8c302ce09 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -167,9 +167,9 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool { if graph.stopCompile(): break var n = parseTopLevelStmt(p) if n.kind == nkEmpty: break - if sfSystemModule notin module.flags and + if (sfSystemModule notin module.flags and ({sfNoForward, sfReorder} * module.flags != {} or - codeReordering in graph.config.features): + codeReordering in graph.config.features)): # read everything, no streaming possible var sl = newNodeI(nkStmtList, n.info) sl.add n diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 7417845c0..cd6fb2a53 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -42,6 +42,10 @@ proc cmpPaths*(x, y: AbsoluteDir): int {.borrow.} proc createDir*(x: AbsoluteDir) {.borrow.} +proc toAbsoluteDir*(path: string): AbsoluteDir = + result = if path.isAbsolute: AbsoluteDir(path) + else: AbsoluteDir(getCurrentDir() / path) + proc `$`*(x: AnyPath): string = x.string when true: diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 9b3c66104..addc9e9f3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -21,7 +21,7 @@ const const procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, - wCompilerProc, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, + wCompilerProc, wNonReloadable, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe, wOverride, @@ -31,7 +31,7 @@ const templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty, wDelegator, wExportNims, wUsed, wPragma} macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc, - wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern, + wNodecl, wMagic, wNosideeffect, wCompilerProc, wNonReloadable, wCore, wDeprecated, wExtern, wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator, wExportNims, wUsed} iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect, @@ -888,6 +888,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, cppDefine(c.graph.config, sym.name.s) recordPragma(c, it, "cppdefine", sym.name.s) if sfFromGeneric notin sym.flags: markCompilerProc(c, sym) + of wNonReloadable: + sym.flags.incl sfNonReloadable of wProcVar: noVal(c, it) incl(sym.flags, sfProcvar) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f66ec7062..46ae4941a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2535,19 +2535,22 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "{}")), flags) of nkPragmaExpr: var - expr = n[0] pragma = n[1] pragmaName = considerQuotedIdent(c, pragma[0]) flags = flags + finalNodeFlags: TNodeFlags = {} case whichKeyword(pragmaName) of wExplain: flags.incl efExplain + of wExecuteOnReload: + finalNodeFlags.incl nfExecuteOnReload else: # what other pragmas are allowed for expressions? `likely`, `unlikely` invalidPragma(c, n) result = semExpr(c, n[0], flags) + result.flags.incl finalNodeFlags of nkPar, nkTupleConstr: case checkPar(c, n) of paNone: result = errorNode(c, n) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 246871928..5fb2fcd65 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -15,8 +15,6 @@ import nversion, platform, math, msgs, os, condsyms, idents, renderer, types, commands, magicsys, modulegraphs, strtabs, lineinfos -import system/indexerrors - proc newIntNodeT*(intVal: BiggestInt, n: PNode; g: ModuleGraph): PNode = case skipTypes(n.typ, abstractVarRange).kind of tyInt: diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 95240867f..4247c6caa 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -260,9 +260,13 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = else: for i in 0..<t.len: c.hashType(t.sons[i], flags) c &= char(t.callConv) - if CoType notin flags: - if tfNoSideEffect in t.flags: c &= ".noSideEffect" - if tfThread in t.flags: c &= ".thread" + # purity of functions doesn't have to affect the mangling (which is in fact + # problematic for HCR - someone could have cached a pointer to another + # function which changes its purity and suddenly the cached pointer is danglign) + # IMHO anything that doesn't affect the overload resolution shouldn't be part of the mangling... + # if CoType notin flags: + # if tfNoSideEffect in t.flags: c &= ".noSideEffect" + # if tfThread in t.flags: c &= ".thread" if tfVarargs in t.flags: c &= ".varargs" of tyArray: c &= char(t.kind) @@ -344,6 +348,12 @@ proc hashOwner*(s: PSym): SigHash = md5Final c, result.Md5Digest +proc sigHash*(s: PSym): SigHash = + if s.kind in routineKinds and s.typ != nil: + result = hashProc(s) + else: + result = hashNonProc(s) + proc idOrSig*(s: PSym, currentModule: string, sigCollisions: var CountTable[SigHash]): Rope = if s.kind in routineKinds and s.typ != nil: diff --git a/compiler/transf.nim b/compiler/transf.nim index a26598094..ec390da9a 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -814,8 +814,9 @@ proc transformExceptBranch(c: PTransf, n: PNode): PTransNode = proc dontInlineConstant(orig, cnst: PNode): bool {.inline.} = # symbols that expand to a complex constant (array, etc.) should not be # inlined, unless it's the empty array: - result = orig.kind == nkSym and cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and - cnst.len != 0 + result = orig.kind == nkSym and + cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and + cnst.len != 0 proc commonOptimizations*(g: ModuleGraph; c: PSym, n: PNode): PNode = result = n @@ -1023,10 +1024,15 @@ proc transform(c: PTransf, n: PNode): PTransNode = when false: if oldDeferAnchor != nil: c.deferAnchor = oldDeferAnchor - var cnst = getConstExpr(c.module, PNode(result), c.graph) - # we inline constants if they are not complex constants: - if cnst != nil and not dontInlineConstant(n, cnst): - result = PTransNode(cnst) # do not miss an optimization + # Constants can be inlined here, but only if they cannot result in a cast + # in the back-end (e.g. var p: pointer = someProc) + let exprIsPointerCast = n.kind in {nkCast, nkConv, nkHiddenStdConv} and + n.typ.kind == tyPointer + if not exprIsPointerCast: + var cnst = getConstExpr(c.module, PNode(result), c.graph) + # we inline constants if they are not complex constants: + if cnst != nil and not dontInlineConstant(n, cnst): + result = PTransNode(cnst) # do not miss an optimization proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode = # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip diff --git a/compiler/vm.nim b/compiler/vm.nim index 7493b008d..0922f7353 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -11,12 +11,12 @@ ## An instruction is 1-3 int32s in memory, it is a register based VM. import ast except getstr -import system/indexerrors import strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, parser, vmdeps, idents, trees, renderer, options, transf, parseutils, - vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl + vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl, + sighashes from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate @@ -1492,6 +1492,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = regs[ra].node.strVal = a.sym.name.s else: stackTrace(c, tos, pc, errFieldXNotFound & "strVal") + of opcNSigHash: + decodeB(rkNode) + createStr regs[ra] + if regs[rb].node.kind != nkSym: + stackTrace(c, tos, pc, "node is not a symbol") + else: + regs[ra].node.strVal = $sigHash(regs[rb].node.sym) of opcSlurp: decodeB(rkNode) createStr regs[ra] diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index f3fac7c69..3b6a44deb 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -92,6 +92,7 @@ type opcNIdent, opcNGetType, opcNStrVal, + opcNSigHash, opcNSetIntVal, opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetType, opcNSetStrVal, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index e5e6d8735..ab2ac8707 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1224,6 +1224,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.freeTemp(tmp) #genUnaryABC(c, n, dest, opcNGetType) of mNStrVal: genUnaryABC(c, n, dest, opcNStrVal) + of mNSigHash: genUnaryABC(c, n , dest, opcNSigHash) of mNSetIntVal: unused(c, n, dest) genBinaryStmt(c, n, opcNSetIntVal) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6f78c9d6f..7a24a5812 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -53,6 +53,7 @@ type wFastcall, wClosure, wNoconv, wOn, wOff, wChecks, wRangechecks, wBoundchecks, wOverflowchecks, wNilchecks, wFloatchecks, wNanChecks, wInfChecks, wMoveChecks, + wNonReloadable, wExecuteOnReload, wAssertions, wPatterns, wWarnings, wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags, wDeadCodeElimUnused, # deprecated, dead code elim always happens @@ -142,6 +143,7 @@ const "noconv", "on", "off", "checks", "rangechecks", "boundchecks", "overflowchecks", "nilchecks", "floatchecks", "nanchecks", "infchecks", "movechecks", + "nonreloadable", "executeonreload", "assertions", "patterns", "warnings", "hints", "optimization", "raises", "writes", "reads", "size", "effects", "tags", diff --git a/doc/advopt.txt b/doc/advopt.txt index 58ed9edd0..b0c23e391 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -18,6 +18,7 @@ Advanced commands: Advanced options: -o:FILE, --out:FILE set the output filename + --outdir:DIR set the path where the output file will be written --stdout:on|off output to stdout --colors:on|off turn compiler messages coloring on|off --listFullPaths:on|off list full paths in messages diff --git a/doc/nimc.rst b/doc/nimc.rst index 7c07e09d8..fca3ed1ed 100644 --- a/doc/nimc.rst +++ b/doc/nimc.rst @@ -416,44 +416,92 @@ is raised. Hot code reloading ------------------ -**Note:** At the moment hot code reloading is supported only in -JavaScript projects. -The `hotCodeReloading`:idx: option enables special compilation mode where changes in -the code can be applied automatically to a running program. The code reloading -happens at the granularity of an individual module. When a module is reloaded, -Nim will preserve the state of all global variables which are initialized with -a standard variable declaration in the code. All other top level code will be -executed repeatedly on each reload. If you want to prevent this behavior, you -can guard a block of code with the ``once`` construct: +The `hotCodeReloading`:idx: option enables special compilation mode where +changes in the code can be applied automatically to a running program. +The code reloading happens at the granularity of an individual module. +When a module is reloaded, any newly added global variables will be +initialized, but all other top-level code appearing in the module won't +be re-executed and the state of all existing global variables will be +preserved. One can use the special event handlers ``beforeCodeReload`` and +``afterCodeReload`` to reset the state of a particular variable or to force +the execution of certain statements: .. code-block:: Nim - var settings = initTable[string, string]() + var + settings = initTable[string, string]() + lastReload: Time - once: - myInit() + for k, v in loadSettings(): + settings[k] = v - for k, v in loadSettings(): - settings[k] = v + initProgram() -If you want to reset the state of a global variable on each reload, just -re-assign a value anywhere within the top-level code: + afterCodeReload: + lastReload = now() + resetProgramState() + +On each code reload, Nim will first execute all `beforeCodeReload`:idx: +handlers registered in the previous version of the program and then all +`afterCodeReload`:idx handlers appearing in the newly loaded code. Please note +that any handlers appearing in modules that weren't reloaded will also be +executed. To prevent this behavior, one can guard the code with the +`hasModuleChanged()`:idx: API: .. code-block:: Nim - var lastReload: Time + import mydb + + var myCache = initTable[Key, Value]() + + afterCodeReload: + if hasModuleChanged(mydb): + resetCache(myCache) + +The hot code reloading is based on dynamic library hot swapping in the native +targets and direct manipulation of the global namespace in the JavaScript +target. The Nim compiler does not specify the mechanism for detecting the +conditions when the code must be reloaded. Instead, the program code is +expected to call `performCodeReload()`:idx every time it wishes to reload +its code. + +It's expected that most projects will implement the reloading with a suitable +build-system triggered IPC notification mechanism, but a polling solution is +also possible through the provided `hasAnyModuleChanged()`:idx API. + +In order to access ``beforeCodeReload``, ``afterCodeReload``, ``hasModuleChanged`` +or ``hasAnyModuleChanged`` one must import the `hotcodereloading`:idx module. + +**Usage in Native projects:** + +Native projects using the hot code reloading option will be implicitly +compiled with the `-d:useNimRtl` option and they will depend on both +the ``nimrtl`` library and the ``nimhcr`` library which implements the +hot code reloading run-time. + +All modules of the project will be compiled to separate dynamic link +libraries placed in the ``nimcache`` directory. Please note that during +the execution of the program, the hot code reloading run-time will load +only copies of these libraries in order to not interfere with any newly +issued build commands. - lastReload = now() - resetProgramState() +The main module of the program is considered non-reloadable. Please note +that procs from reloadable modules should not appear in the call stack of +program while ``performCodeReload`` is being called. Thus, the main module +is a suitable place for implementing a program loop capable of calling +``performCodeReload``. -**Known limitations:** In the JavaScript target, global variables using the -``codegenDecl`` pragma will be re-initialized on each reload. Please guard the -initialization with a `once` block to work-around this. +Please note that reloading won't be possible when any of the type definitions +in the program has been changed. When closure iterators are used (directly or +through async code), the reloaded refinitions will affect only newly created +instances. Existing iterator instancess will execute their original code to +completion. **Usage in JavaScript projects:** -Once your code is compiled for hot reloading, you can use a framework such -as `LiveReload <http://livereload.com/>` or `BrowserSync <https://browsersync.io/>` -to implement the actual reloading behavior in your project. +Once your code is compiled for hot reloading, the ``nim-livereload`` NPM +package provides a convenient solution for implementing the actual reloading +in the browser using a framework such as [LiveReload](http://livereload.com/) +or [BrowserSync](https://browsersync.io/). DynlibOverride diff --git a/lib/core/hotcodereloading.nim b/lib/core/hotcodereloading.nim new file mode 100644 index 000000000..8b48b3d69 --- /dev/null +++ b/lib/core/hotcodereloading.nim @@ -0,0 +1,27 @@ +when defined(hotcodereloading): + import + macros + + template beforeCodeReload*(body: untyped) = + hcrAddEventHandler(true, proc = body) {.executeOnReload.} + + template afterCodeReload*(body: untyped) = + hcrAddEventHandler(false, proc = body) {.executeOnReload.} + + macro hasModuleChanged*(module: typed): untyped = + if module.kind != nnkSym or module.symKind != nskModule: + error "hasModuleChanged expects a module symbol", module + return newCall(bindSym"hcrHasModuleChanged", newLit(module.signatureHash)) + + proc hasAnyModuleChanged*(): bool = hcrReloadNeeded() + + when not defined(JS): + template performCodeReload* = hcrPerformCodeReload() + else: + template performCodeReload* = discard +else: + template beforeCodeReload*(body: untyped) = discard + template afterCodeReload*(body: untyped) = discard + template hasModuleChanged*(module: typed): bool = false + proc hasAnyModuleChanged*(): bool = false + template performCodeReload*() = discard diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 461afb963..8e6b93a11 100644 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -347,6 +347,13 @@ object doAssert(dumpTypeImpl(b) == t) doAssert(dumpTypeImpl(c) == t) +when defined(nimHasSignatureHashInMacro): + proc signatureHash*(n: NimNode): string {.magic: "NSigHash", noSideEffect.} + ## Returns a stable identifier derived from the signature of a symbol. + ## The signature combines many factors such as the type of the symbol, + ## the owning module of the symbol and others. The same identifier is + ## used in the back-end to produce the mangled symbol name. + proc getTypeImpl*(n: typedesc): NimNode {.magic: "NGetType", noSideEffect.} ## Version of ``getTypeImpl`` which takes a ``typedesc``. diff --git a/lib/core/strs.nim b/lib/core/strs.nim index ccbde76fe..e55c88493 100644 --- a/lib/core/strs.nim +++ b/lib/core/strs.nim @@ -125,7 +125,7 @@ proc cstrToNimstr(str: cstring): NimStringV2 {.compilerRtl.} = if str == nil: toNimStr(str, 0) else: toNimStr(str, str.len) -proc nimToCStringConv(s: NimStringV2): cstring {.compilerProc, inline.} = +proc nimToCStringConv(s: NimStringV2): cstring {.compilerProc, nonReloadable, inline.} = if s.len == 0: result = cstring"" else: result = cstring(unsafeAddr s.p.data) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index d6dd16b54..fe958c7f5 100644 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -27,8 +27,6 @@ include "system/inclrtl.nim" include "system/hti.nim" -import system/indexerrors - {.pop.} type diff --git a/lib/nimbase.h b/lib/nimbase.h index ba4273726..9fd475c85 100644 --- a/lib/nimbase.h +++ b/lib/nimbase.h @@ -354,7 +354,12 @@ typedef NU8 NU; # endif #endif +// for now there isn't an easy way for C code to reach the program result +// when hot code reloading is ON - users will have to: +// load the nimhcr.dll, get the hcrGetGlobal proc from there and use it +#ifndef NIM_HOT_CODE_RELOADING extern NI nim_program_result; +#endif typedef float NF32; typedef double NF64; diff --git a/lib/nimhcr.nim b/lib/nimhcr.nim new file mode 100644 index 000000000..f3afac347 --- /dev/null +++ b/lib/nimhcr.nim @@ -0,0 +1,652 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2018 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This is the Nim hot code reloading run-time for the native targets. +## +## This minimal dynamic library is not subject to reloading when the +## `hotCodeReloading` build mode is enabled. It's responsible for providing +## a permanent memory location for all globals and procs within a program +## and orchestrating the reloading. For globals, this is easily achieved +## by storing them on the heap. For procs, we produce on the fly simple +## trampolines that can be dynamically overwritten to jump to a different +## target. In the host program, all globals and procs are first registered +## here with ``hcrRegisterGlobal`` and ``hcrRegisterProc`` and then the +## returned permanent locations are used in every reference to these symbols +## onwards. +## +## Detailed description: +## +## When code is compiled with the hotCodeReloading option for native targets +## a couple of things happen for all modules in a project: +## - the useNimRtl option is forced (including when building the HCR runtime too) +## - all modules of a target get built into separate shared libraries +## - the smallest granularity of reloads is modules +## - for each .c (or .cpp) in the corresponding nimcache folder of the project +## a shared object is built with the name of the source file + DLL extension +## - only the main module produces whatever the original project type intends +## (again in nimcache) and is then copied to its original destination +## - linking is done in parallel - just like compilation +## - function calls to functions from the same project go through function pointers: +## - with a few exceptions - see the nonReloadable pragma +## - the forward declarations of the original functions become function +## pointers as static globals with the same names +## - the original function definitions get suffixed with <name>_actual +## - the function pointers get initialized with the address of the corresponding +## function in the DatInit of their module through a call to either hcrRegisterProc +## or hcrGetProc. When being registered, the <name>_actual address is passed to +## hcrRegisterProc and a permanent location is returned and assigned to the pointer. +## This way the implementation (<name>_actual) can change but the address for it +## will be the same - this works by just updating a jump instruction (trampoline). +## For functions from other modules hcrGetProc is used (after they are registered). +## - globals are initialized only once and their state is preserved +## - including locals with the {.global.} pragma +## - their definitions are changed into pointer definitions which are initialized +## in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the +## size of the type that this HCR runtime should allocate) and a bool is returned +## which when true triggers the initialization code for the global (only once). +## Globals from other modules: a global pointer coupled with a hcrGetGlobal call. +## - globals which have already been initialized cannot have their values changed +## by changing their initialization - use a handler or some other mechanism +## - new globals can be introduced when reloading +## - top-level code (global scope) is executed only once - at the first module load +## - the runtime knows every symbol's module owner (globals and procs) +## - both the RTL and HCR shared libraries need to be near the program for execution +## - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS) +## - the main module is responsible for initializing the HCR runtime +## - the main module loads the RTL and HCR shared objects +## - after that a call to hcrInit() is done in the main module which triggers +## the loading of all modules the main one imports, and doing that for the +## dependencies of each module recursively. Basically a DFS traversal. +## - then initialization takes place with several passes over all modules: +## - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc +## - HcrCreateTypeInfos - creates globals which will be referenced in the next pass +## - DatInit - usual dat init + register/get procs and get globals +## - Init - it does the following multiplexed operations: +## - register globals (if already registered - then just retrieve pointer) +## - execute top level scope (only if loaded for the first time) +## - when modules are loaded the originally built shared libraries get copied in +## the same folder and the copies are loaded instead of the original files +## - a module import tree is built in the runtime (and maintained when reloading) +## - hcrPerformCodeReload +## - named `performCodeReload`, requires the hotcodereloading module +## - explicitly called by the user - the current active callstack shouldn't contain +## any functions which are defined in modules that will be reloaded (or crash!). +## The reason is that old dynalic libraries get unloaded. +## Example: +## if A is the main module and it imports B, then only B is reloadable and only +## if when calling hcrPerformCodeReload there is no function defined in B in the +## current active callstack at the point of the call (it has to be done from A) +## - for reloading to take place the user has to have rebuilt parts of the application +## without changes affecting the main module in any way - it shouldn't be rebuilt. +## - to determine what needs to be reloaded the runtime starts traversing the import +## tree from the root and checks the timestamps of the loaded shared objects +## - modules that are no longer referenced are unloaded and cleaned up properly +## - symbols (procs/globals) that have been removed in the code are also cleaned up +## - so changing the init of a global does nothing, but removing it, reloading, +## and then re-introducing it with a new initializer works +## - new modules can be imported, and imports can also be reodereded/removed +## - hcrReloadNeeded() can be used to determine if any module needs reloading +## - named `hasAnyModuleChanged`, requires the hotcodereloading module +## - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload +## - require the hotcodereloading module +## - such handlers can be added and removed +## - before each reload all "beforeCodeReload" handlers are executed and after +## that all handlers (including "after") from the particular module are deleted +## - the order of execution is the same as the order of top-level code execution. +## Example: if A imports B which imports C, then all handlers in C will be executed +## first (from top to bottom) followed by all from B and lastly all from A +## - after the reload all "after" handlers are executed the same way as "before" +## - the handlers for a reloaded module are always removed when reloading and then +## registered when the top-level scope is executed (thanks to `executeOnReload`) +## +## TODO - after first merge in upstream Nim: +## +## - profile +## - build speed with and without hot code reloading - difference should be small +## - runtime degradation of HCR-enabled code - important!!! +## - ARM support for the trampolines +## - investigate: +## - rethink the closure iterators +## - ability to keep old versions of dynamic libraries alive +## - because of async server code +## - perhaps with refcounting of .dlls for unfinished closures +## - linking with static libs +## - all shared objects for each module will (probably) have to link to them +## - state in static libs gets duplicated +## - linking is slow and therefore iteration time suffers +## - have just a single .dll for all .nim files and bulk reload? +## - think about the compile/link/passC/passL/emit/injectStmt pragmas +## - if a passC pragma is introduced (either written or dragged in by a new +## import) the whole command line for compilation changes - for example: +## winlean.nim: {.passC: "-DWIN32_LEAN_AND_MEAN".} +## - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*' +## - everything thread-local related +## - tests +## - add a new travis build matrix entry which builds everything with HCR enabled +## - currently building with useNimRtl is problematic - lots of problems... +## - how to supply the nimrtl/nimhcr shared objects to all test binaries...? +## - think about building to C++ instead of only to C - added type safety +## - run tests through valgrind and the sanitizers! of HUGE importance! +## +## TODO - nice to have cool stuff: +## +## - separate handling of global state for much faster reloading and manipulation +## - imagine sliders in an IDE for tweaking variables +## - perhaps using shared memory +## - multi-dll projects - how everything can be reloaded..? +## - a single HCR instance shared across multiple .dlls +## - instead of having to call hcrPerformCodeReload from a function in each dll +## - which currently renders the main module of each dll not reloadable +## - ability to check with the current callstack if a reload is "legal" +## - if it is in any function which is in a module about to be reloaded ==> error +## - pragma annotations for files - to be excluded from dll shenanigans +## - for such file-global pragmas look at codeReordering or injectStmt +## - how would the initialization order be kept? messy... +## - per function exclude pragmas would be TOO messy and hard... +## - C code calling stable exportc interface of nim code (for bindings) +## - generate proxy functions with the stable names +## - in a non-reloadable part (the main binary) that call the function pointers +## - parameter passing/forwarding - how? use the same trampoline jumping? +## - extracting the dependencies for these stubs/proxies will be hard... +## - changing memory layout of types - detecting this..? +## - implement with registerType() call to HCR runtime...? +## - and checking if a previously registered type matches +## - issue an error +## - or let the user handle this by transferring the state properly +## - perhaps in the before/afterCodeReload handlers +## - optimization: calls to procs within a module (+inlined) to use the _actual versions +## - implement executeOnReload for global vars too - not just statements (and document!) +## - cleanup at shutdown - freeing all globals +## +## TODO - unimportant: +## +## - have a "bad call" trampoline that all no-longer-present functions are routed to call there +## - so the user gets some error msg if he calls a dangling pointer instead of a crash +## - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate +## - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h) +## - proper .json build file - but the format is different... multiple link commands... +## - avoid registering globals on each loop when using an iterator in global scope +## +## TODO - REPL: +## - proper way (as proposed by Zahary): +## - parse the input code and put everything in global scope except for +## statements with side effects only - those go in afterCodeReload blocks +## - my very hacky idea: just append to a closure iterator the new statements +## followed by a yield statement. So far I can think of 2 problems: +## - import and some other code cannot be written inside of a proc - +## has to be parsed and extracted in the outer scope +## - when new variables are created they are actually locals to the closure +## so the struct for the closure state grows in memory, but it has already +## been allocated when the closure was created with the previous smaller size. +## That would lead to working with memory outside of the initially allocated +## block. Perhaps something can be done about this - some way of re-allocating +## the state and transferring the old... + +when not defined(JS) and (defined(hotcodereloading) or + defined(createNimHcr) or + defined(testNimHcr)): + const + dllExt = when defined(windows): "dll" + elif defined(macosx): "dylib" + else: "so" + type + HcrProcGetter* = proc (libHandle: pointer, procName: cstring): pointer {.nimcall.} + HcrGcMarkerProc = proc () {.nimcall.} + HcrModuleInitializer* = proc () {.nimcall.} + +when defined(createNimHcr): + when system.appType != "lib": + {.error: "This file has to be compiled as a library!".} + + import os, tables, sets, times, strutils, reservedmem, dynlib + + template trace(args: varargs[untyped]) = + when defined(testNimHcr) or defined(traceHcr): + echo args + + proc sanitize(arg: Time): string = + when defined(testNimHcr): return "<time>" + else: return $arg + + proc sanitize(arg: string|cstring): string = + when defined(testNimHcr): return ($arg).splitFile.name.splitFile.name + else: return $arg + + {.pragma: nimhcr, compilerProc, exportc, dynlib.} + + when hostCPU in ["i386", "amd64"]: + type + ShortJumpInstruction {.packed.} = object + opcode: byte + offset: int32 + + LongJumpInstruction {.packed.} = object + opcode1: byte + opcode2: byte + offset: int32 + absoluteAddr: pointer + + proc writeJump(jumpTableEntry: ptr LongJumpInstruction, targetFn: pointer) = + let + jumpFrom = jumpTableEntry.shift(sizeof(ShortJumpInstruction)) + jumpDistance = distance(jumpFrom, targetFn) + + if abs(jumpDistance) < 0x7fff0000: + let shortJump = cast[ptr ShortJumpInstruction](jumpTableEntry) + shortJump.opcode = 0xE9 # relative jump + shortJump.offset = int32(jumpDistance) + else: + jumpTableEntry.opcode1 = 0xff # indirect absolute jump + jumpTableEntry.opcode2 = 0x25 + when hostCPU == "i386": + # on x86 we write the absolute address of the following pointer + jumpTableEntry.offset = cast[int32](addr jumpTableEntry.absoluteAddr) + else: + # on x64, we use a relative address for the same location + jumpTableEntry.offset = 0 + jumpTableEntry.absoluteAddr = targetFn + + elif hostCPU == "arm": + const jumpSize = 8 + elif hostCPU == "arm64": + const jumpSize = 16 + + const defaultJumpTableSize = case hostCPU + of "i386": 50 + of "amd64": 500 + else: 50 + + let jumpTableSizeStr = getEnv("HOT_CODE_RELOADING_JUMP_TABLE_SIZE") + let jumpTableSize = if jumpTableSizeStr.len > 0: parseInt(jumpTableSizeStr) + else: defaultJumpTableSize + + # TODO: perhaps keep track of free slots due to removed procs using a free list + var jumpTable = ReservedMemSeq[LongJumpInstruction].init( + memStart = cast[pointer](0x10000000), + maxLen = jumpTableSize * 1024 * 1024 div sizeof(LongJumpInstruction), + accessFlags = memExecReadWrite) + + type + ProcSym = object + jump: ptr LongJumpInstruction + gen: int + + GlobalVarSym = object + p: pointer + markerProc: HcrGcMarkerProc + gen: int + + ModuleDesc = object + procs: Table[string, ProcSym] + globals: Table[string, GlobalVarSym] + imports: seq[string] + handle: LibHandle + hash: string + gen: int + lastModification: Time + handlers: seq[tuple[isBefore: bool, cb: proc ()]] + + proc newModuleDesc(): ModuleDesc = + result.procs = initTable[string, ProcSym]() + result.globals = initTable[string, GlobalVarSym]() + result.handle = nil + result.gen = -1 + result.lastModification = low(Time) + + # the global state necessary for traversing and reloading the module import tree + var modules = initTable[string, ModuleDesc]() + var root: string + var system: string + var mainDatInit: HcrModuleInitializer + var generation = 0 + + # necessary for queries such as "has module X changed" - contains all but the main module + var hashToModuleMap = initTable[string, string]() + + # necessary for registering handlers and keeping them up-to-date + var currentModule: string + + # supplied from the main module - used by others to initialize pointers to this runtime + var hcrDynlibHandle: pointer + var getProcAddr: HcrProcGetter + + proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} = + trace " register proc: ", module.sanitize, " ", name + # Please note: We must allocate a local copy of the strings, because the supplied + # `cstring` will reside in the data segment of a DLL that will be later unloaded. + let name = $name + let module = $module + + var jumpTableEntryAddr: ptr LongJumpInstruction + + modules[module].procs.withValue(name, p): + trace " update proc: ", name + jumpTableEntryAddr = p.jump + p.gen = generation + do: + let len = jumpTable.len + jumpTable.setLen(len + 1) + jumpTableEntryAddr = addr jumpTable[len] + modules[module].procs[name] = ProcSym(jump: jumpTableEntryAddr, gen: generation) + + writeJump jumpTableEntryAddr, fn + return jumpTableEntryAddr + + proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} = + trace " get proc: ", module.sanitize, " ", name + return modules[$module].procs[$name].jump + + proc hcrRegisterGlobal*(module: cstring, + name: cstring, + size: Natural, + gcMarker: HcrGcMarkerProc, + outPtr: ptr pointer): bool {.nimhcr.} = + trace " register global: ", module.sanitize, " ", name + # Please note: We must allocate local copies of the strings, because the supplied + # `cstring` will reside in the data segment of a DLL that will be later unloaded. + # Also using a ptr pointer instead of a var pointer (an output parameter) + # because for the C++ backend var parameters use references and in this use case + # it is not possible to cast an int* (for example) to a void* and then pass it + # to void*& since the casting yields an rvalue and references bind only to lvalues. + let name = $name + let module = $module + + modules[module].globals.withValue(name, global): + trace " update global: ", name + outPtr[] = global.p + global.gen = generation + global.markerProc = gcMarker + return false + do: + outPtr[] = alloc0(size) + modules[module].globals[name] = GlobalVarSym(p: outPtr[], + gen: generation, + markerProc: gcMarker) + return true + + proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} = + trace " get global: ", module.sanitize, " ", name + return modules[$module].globals[$name].p + + proc getListOfModules(cstringArray: ptr pointer): seq[string] = + var curr = cast[ptr cstring](cstringArray) + while len(curr[]) > 0: + result.add($curr[]) + curr = cast[ptr cstring](cast[int64](curr) + sizeof(ptr cstring)) + + template cleanup(collection, body) = + var toDelete: seq[string] + for name, data in collection.pairs: + if data.gen < generation: + toDelete.add(name) + trace "HCR Cleaning ", astToStr(collection), " :: ", name, " ", data.gen + for name {.inject.} in toDelete: + body + + proc cleanupGlobal(module: string, name: string) = + var g: GlobalVarSym + if modules[module].globals.take(name, g): + dealloc g.p + + proc cleanupSymbols(module: string) = + cleanup modules[module].globals: + cleanupGlobal(module, name) + + cleanup modules[module].procs: + modules[module].procs.del(name) + + proc unloadDll(name: string) = + if modules[name].handle != nil: + unloadLib(modules[name].handle) + + proc loadDll(name: cstring) {.nimhcr.} = + let name = $name + trace "HCR LOADING: ", name.sanitize + if modules.contains(name): + unloadDll(name) + else: + modules.add(name, newModuleDesc()) + + let copiedName = name & ".copy." & dllExt + copyFile(name, copiedName) + + let lib = loadLib(copiedName) + assert lib != nil + modules[name].handle = lib + modules[name].gen = generation + modules[name].lastModification = getLastModificationTime(name) + + # update the list of imports by the module + let getImportsProc = cast[proc (): ptr pointer {.nimcall.}]( + checkedSymAddr(lib, "HcrGetImportedModules")) + modules[name].imports = getListOfModules(getImportsProc()) + # get the hash of the module + let getHashProc = cast[proc (): cstring {.nimcall.}]( + checkedSymAddr(lib, "HcrGetSigHash")) + modules[name].hash = $getHashProc() + hashToModuleMap[modules[name].hash] = name + + # Remove handlers for this module if reloading - they will be re-registered. + # In order for them to be re-registered we need to de-register all globals + # that trigger the registering of handlers through calls to hcrAddEventHandler + modules[name].handlers.setLen(0) + + proc initHcrData(name: cstring) {.nimhcr.} = + trace "HCR Hcr init: ", name.sanitize + cast[proc (h: pointer, gpa: HcrProcGetter) {.nimcall.}]( + checkedSymAddr(modules[$name].handle, "HcrInit000"))(hcrDynlibHandle, getProcAddr) + + proc initTypeInfoGlobals(name: cstring) {.nimhcr.} = + trace "HCR TypeInfo globals init: ", name.sanitize + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "HcrCreateTypeInfos"))() + + proc initPointerData(name: cstring) {.nimhcr.} = + trace "HCR Dat init: ", name.sanitize + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "DatInit000"))() + + proc initGlobalScope(name: cstring) {.nimhcr.} = + trace "HCR Init000: ", name.sanitize + # set the currently inited module - necessary for registering the before/after HCR handlers + currentModule = $name + cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "Init000"))() + + var modulesToInit: seq[string] = @[] + var allModulesOrderedByDFS: seq[string] = @[] + + proc recursiveDiscovery(dlls: seq[string]) = + for curr in dlls: + if modules.contains(curr): + # skip updating modules that have already been updated to the latest generation + if modules[curr].gen >= generation: + trace "HCR SKIP: ", curr.sanitize, " gen is already: ", modules[curr].gen + continue + # skip updating an unmodified module but continue traversing its dependencies + if modules[curr].lastModification >= getLastModificationTime(curr): + trace "HCR SKIP (not modified): ", curr.sanitize, " ", modules[curr].lastModification.sanitize + # update generation so module doesn't get collected + modules[curr].gen = generation + # recurse to imported modules - they might be changed + recursiveDiscovery(modules[curr].imports) + allModulesOrderedByDFS.add(curr) + continue + loadDll(curr) + # first load all dependencies of the current module and init it after that + recursiveDiscovery(modules[curr].imports) + + allModulesOrderedByDFS.add(curr) + modulesToInit.add(curr) + + proc initModules() = + # first init the pointers to hcr functions and also do the registering of typeinfo globals + for curr in modulesToInit: + initHcrData(curr) + initTypeInfoGlobals(curr) + # for now system always gets fully inited before any other module (including when reloading) + initPointerData(system) + initGlobalScope(system) + # proceed with the DatInit calls - for all modules - including the main one! + for curr in allModulesOrderedByDFS: + if curr != system: + initPointerData(curr) + mainDatInit() + # execute top-level code (in global scope) + for curr in modulesToInit: + if curr != system: + initGlobalScope(curr) + # cleanup old symbols which are gone now + for curr in modulesToInit: + cleanupSymbols(curr) + + proc hcrInit*(moduleList: ptr pointer, main, sys: cstring, + datInit: HcrModuleInitializer, handle: pointer, gpa: HcrProcGetter) {.nimhcr.} = + trace "HCR INITING: ", main.sanitize, " gen: ", generation + # initialize globals + root = $main + system = $sys + mainDatInit = datInit + hcrDynlibHandle = handle + getProcAddr = gpa + # the root is already added and we need it because symbols from it will also be registered in the HCR system + modules[root].imports = getListOfModules(moduleList) + modules[root].gen = high(int) # something huge so it doesn't get collected + # recursively initialize all modules + recursiveDiscovery(modules[root].imports) + initModules() + # the next module to be inited will be the root + currentModule = root + + proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} = + let module = hashToModuleMap[moduleHash] + return modules[module].lastModification < getLastModificationTime(module) + + proc hcrReloadNeeded*(): bool {.nimhcr.} = + for hash, _ in hashToModuleMap: + if hcrHasModuleChanged(hash): + return true + return false + + proc hcrPerformCodeReload*() {.nimhcr.} = + if not hcrReloadNeeded(): + trace "HCR - no changes" + return + + # We disable the GC during the reload, because the reloading procedures + # will replace type info objects and GC marker procs. This seems to create + # problems when the GC is executed while the reload is underway. + # Future versions of NIMHCR won't use the GC, because all globals and the + # metadata needed to access them will be placed in shared memory, so they + # can be manipulted from external programs without reloading. + GC_disable() + defer: GC_enable() + + inc(generation) + trace "HCR RELOADING: ", generation + + var traversedHandlerModules = initSet[string]() + + proc recursiveExecuteHandlers(isBefore: bool, module: string) = + # do not process an already traversed module + if traversedHandlerModules.containsOrIncl(module): return + traversedHandlerModules.incl module + # first recurse to do a DFS traversal + for curr in modules[module].imports: + recursiveExecuteHandlers(isBefore, curr) + # and then execute the handlers - from leaf modules all the way up to the root module + for curr in modules[module].handlers: + if curr.isBefore == isBefore: + curr.cb() + + # first execute the before reload handlers + traversedHandlerModules.clear() + recursiveExecuteHandlers(true, root) + + # do the reloading + modulesToInit = @[] + allModulesOrderedByDFS = @[] + recursiveDiscovery(modules[root].imports) + initModules() + + # execute the after reload handlers + traversedHandlerModules.clear() + recursiveExecuteHandlers(false, root) + + # collecting no longer referenced modules - based on their generation + cleanup modules: + cleanupSymbols(name) + unloadDll(name) + hashToModuleMap.del(modules[name].hash) + modules.del(name) + + proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} = + modules[currentModule].handlers.add( + (isBefore: isBefore, cb: cb)) + + proc hcrAddModule*(module: cstring) {.nimhcr.} = + if not modules.contains($module): + modules.add($module, newModuleDesc()) + + proc hcrGeneration*(): int {.nimhcr.} = + generation + + proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} = + # This is gcsafe, because it will be registered + # only in the GC of the main thread. + {.gcsafe.}: + for _, module in modules: + for _, global in module.globals: + if global.markerProc != nil: + global.markerProc() + +elif defined(hotcodereloading) or defined(testNimHcr): + when not defined(JS): + const + nimhcrLibname = when defined(windows): "nimhcr." & dllExt + elif defined(macosx): "libnimhcr." & dllExt + else: "libnimhcr." & dllExt + + {.pragma: nimhcr, compilerProc, importc, dynlib: nimhcrLibname.} + + proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} + + proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} + + proc hcrRegisterGlobal*(module: cstring, name: cstring, size: Natural, + gcMarker: HcrGcMarkerProc, outPtr: ptr pointer): bool {.nimhcr.} + proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} + + proc hcrInit*(moduleList: ptr pointer, + main, sys: cstring, + datInit: HcrModuleInitializer, + handle: pointer, + gpa: HcrProcGetter) {.nimhcr.} + + proc hcrAddModule*(module: cstring) {.nimhcr.} + + proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} + + proc hcrReloadNeeded*(): bool {.nimhcr.} + + proc hcrPerformCodeReload*() {.nimhcr.} + + proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} + + proc hcrMarkGlobals*() {.nimhcr, nimcall, gcsafe.} + + when declared(nimRegisterGlobalMarker): + nimRegisterGlobalMarker(hcrMarkGlobals) + + else: + proc hcrHasModuleChanged*(moduleHash: string): bool = + # TODO + false + + proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) = + # TODO + discard + diff --git a/lib/nimhcr.nim.cfg b/lib/nimhcr.nim.cfg new file mode 100644 index 000000000..282bec27c --- /dev/null +++ b/lib/nimhcr.nim.cfg @@ -0,0 +1,5 @@ +--app:lib +--threads:on +-d:useNimRtl +-d:createNimHcr + diff --git a/lib/nimrtl.nim.cfg b/lib/nimrtl.nim.cfg index b60de183a..a5cec596f 100644 --- a/lib/nimrtl.nim.cfg +++ b/lib/nimrtl.nim.cfg @@ -1,5 +1,6 @@ # The RTL.dll needs to be compiled with these options! --app:lib +--threads:on --define:createNimRtl diff --git a/lib/pure/collections/sharedstrings.nim b/lib/pure/collections/sharedstrings.nim index ca52ec63c..17e2e5888 100644 --- a/lib/pure/collections/sharedstrings.nim +++ b/lib/pure/collections/sharedstrings.nim @@ -12,8 +12,6 @@ type UncheckedCharArray = UncheckedArray[char] -import system/indexerrors - type Buffer = ptr object refcount: int diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 0b9c8babc..fec4ed843 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -47,7 +47,7 @@ include "system/inclrtl" import - strutils, pathnorm, system/indexerrors + strutils, pathnorm const weirdTarget = defined(nimscript) or defined(js) diff --git a/lib/pure/reservedmem.nim b/lib/pure/reservedmem.nim new file mode 100644 index 000000000..22e2b5096 --- /dev/null +++ b/lib/pure/reservedmem.nim @@ -0,0 +1,241 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Nim Contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## :Authors: Zahary Karadjov +## +## This module provides utilities for reserving a portions of the +## address space of a program without consuming physical memory. +## It can be used to implement a dynamically resizable buffer that +## is guaranteed to remain in the same memory location. The buffer +## will be able to grow up to the size of the initially reserved +## portion of the address space. + +from ospaths import raiseOSError, osLastError + +template distance*(lhs, rhs: pointer): int = + cast[int](rhs) - cast[int](lhs) + +template shift*(p: pointer, distance: int): pointer = + cast[pointer](cast[int](p) + distance) + +type + MemAccessFlags* = int + + ReservedMem* = object + memStart: pointer + usedMemEnd: pointer + committedMemEnd: pointer + memEnd: pointer + maxCommittedAndUnusedPages: int + accessFlags: MemAccessFlags + + ReservedMemSeq*[T] = object + mem: ReservedMem + +when defined(windows): + import winlean + + type + SYSTEM_INFO {.final, pure.} = object + u1: uint32 + dwPageSize: uint32 + lpMinimumApplicationAddress: pointer + lpMaximumApplicationAddress: pointer + dwActiveProcessorMask: ptr uint32 + dwNumberOfProcessors: uint32 + dwProcessorType: uint32 + dwAllocationGranularity: uint32 + wProcessorLevel: uint16 + wProcessorRevision: uint16 + + proc getSystemInfo(lpSystemInfo: ptr SYSTEM_INFO) {.stdcall, dynlib: "kernel32", importc: "GetSystemInfo".} + + proc getAllocationGranularity: uint = + var sysInfo: SYSTEM_INFO + getSystemInfo(addr sysInfo) + return uint(sysInfo.dwAllocationGranularity) + + let allocationGranularity = getAllocationGranularity().int + + const + memNoAccess = MemAccessFlags(PAGE_NOACCESS) + memExec* = MemAccessFlags(PAGE_EXECUTE) + memExecRead* = MemAccessFlags(PAGE_EXECUTE_READ) + memExecReadWrite* = MemAccessFlags(PAGE_EXECUTE_READWRITE) + memRead* = MemAccessFlags(PAGE_READONLY) + memReadWrite* = MemAccessFlags(PAGE_READWRITE) + + template check(expr) = + let r = expr + if r == cast[type(r)](0): + raiseOSError(osLastError()) + +else: + import posix + + var MAP_ANONYMOUS {.importc: "MAP_ANONYMOUS", header: "<sys/mman.h>".}: cint + var MAP_NORESERVE {.importc: "MAP_NORESERVE", header: "<sys/mman.h>".}: cint + # var MAP_FIXED_NOREPLACE {.importc: "MAP_FIXED_NOREPLACE", header: "<sys/mman.h>".}: cint + + var SC_PAGESIZE {.importc: "_SC_PAGESIZE", header: "<unistd.h>".}: cint + + let allocationGranularity = sysconf(SC_PAGESIZE) + + let + memNoAccess = MemAccessFlags(PROT_NONE) + memExec* = MemAccessFlags(PROT_EXEC) + memExecRead* = MemAccessFlags(PROT_EXEC or PROT_READ) + memExecReadWrite* = MemAccessFlags(PROT_EXEC or PROT_READ or PROT_WRITE) + memRead* = MemAccessFlags(PROT_READ) + memReadWrite* = MemAccessFlags(PROT_READ or PROT_WRITE) + + template check(expr) = + if not expr: + raiseOSError(osLastError()) + +func nextAlignedOffset(n, alignment: int): int = + result = n + let m = n mod alignment + if m != 0: result += alignment - m + + +when defined(windows): + const + MEM_DECOMMIT = 0x4000 + MEM_RESERVE = 0x2000 + MEM_COMMIT = 0x1000 + proc virtualFree(lpAddress: pointer, dwSize: int, + dwFreeType: int32): cint {.header: "<windows.h>", stdcall, + importc: "VirtualFree".} + proc virtualAlloc(lpAddress: pointer, dwSize: int, flAllocationType, + flProtect: int32): pointer {. + header: "<windows.h>", stdcall, importc: "VirtualAlloc".} + +proc init*(T: type ReservedMem, + maxLen: Natural, + initLen: Natural = 0, + initCommitLen = initLen, + memStart = pointer(nil), + accessFlags = memReadWrite, + maxCommittedAndUnusedPages = 3): ReservedMem = + + assert initLen <= initCommitLen + let commitSize = nextAlignedOffset(initCommitLen, allocationGranularity) + + when defined(windows): + result.memStart = virtualAlloc(memStart, maxLen, MEM_RESERVE, accessFlags.cint) + check result.memStart + if commitSize > 0: + check virtualAlloc(result.memStart, commitSize, MEM_COMMIT, accessFlags.cint) + else: + var allocFlags = MAP_PRIVATE or MAP_ANONYMOUS # or MAP_NORESERVE + # if memStart != nil: + # allocFlags = allocFlags or MAP_FIXED_NOREPLACE + result.memStart = mmap(memStart, maxLen, PROT_NONE, allocFlags, -1, 0) + check result.memStart != MAP_FAILED + if commitSize > 0: + check mprotect(result.memStart, commitSize, cint(accessFlags)) == 0 + + result.usedMemEnd = result.memStart.shift(initLen) + result.committedMemEnd = result.memStart.shift(commitSize) + result.memEnd = result.memStart.shift(maxLen) + result.accessFlags = accessFlags + result.maxCommittedAndUnusedPages = maxCommittedAndUnusedPages + +func len*(m: ReservedMem): int = + distance(m.memStart, m.usedMemEnd) + +func commitedLen*(m: ReservedMem): int = + distance(m.memStart, m.committedMemEnd) + +func maxLen*(m: ReservedMem): int = + distance(m.memStart, m.memEnd) + +proc setLen*(m: var ReservedMem, newLen: int) = + let len = m.len + m.usedMemEnd = m.memStart.shift(newLen) + if newLen > len: + let d = distance(m.committedMemEnd, m.usedMemEnd) + if d > 0: + let commitExtensionSize = nextAlignedOffset(d, allocationGranularity) + when defined(windows): + check virtualAlloc(m.committedMemEnd, commitExtensionSize, + MEM_COMMIT, m.accessFlags.cint) + else: + check mprotect(m.committedMemEnd, commitExtensionSize, m.accessFlags.cint) == 0 + else: + let d = distance(m.usedMemEnd, m.committedMemEnd) - + m.maxCommittedAndUnusedPages * allocationGranularity + if d > 0: + let commitSizeShrinkage = nextAlignedOffset(d, allocationGranularity) + let newCommitEnd = m.committedMemEnd.shift(-commitSizeShrinkage) + + when defined(windows): + check virtualFree(newCommitEnd, commitSizeShrinkage, MEM_DECOMMIT) + else: + check posix_madvise(newCommitEnd, commitSizeShrinkage, + POSIX_MADV_DONTNEED) == 0 + + m.committedMemEnd = newCommitEnd + +proc init*(SeqType: type ReservedMemSeq, + maxLen: Natural, + initLen: Natural = 0, + initCommitLen: Natural = 0, + memStart = pointer(nil), + accessFlags = memReadWrite, + maxCommittedAndUnusedPages = 3): SeqType = + + let elemSize = sizeof(SeqType.T) + result.mem = ReservedMem.init(maxLen * elemSize, + initLen * elemSize, + initCommitLen * elemSize, + memStart, accessFlags, + maxCommittedAndUnusedPages) + +func `[]`*[T](s: ReservedMemSeq[T], pos: Natural): lent T = + let elemAddr = s.mem.memStart.shift(pos * sizeof(T)) + rangeCheck elemAddr < s.mem.usedMemEnd + result = (cast[ptr T](elemAddr))[] + +func `[]`*[T](s: var ReservedMemSeq[T], pos: Natural): var T = + let elemAddr = s.mem.memStart.shift(pos * sizeof(T)) + rangeCheck elemAddr < s.mem.usedMemEnd + result = (cast[ptr T](elemAddr))[] + +func `[]`*[T](s: ReservedMemSeq[T], rpos: BackwardsIndex): lent T = + return s[int(s.len) - int(rpos)] + +func `[]`*[T](s: var ReservedMemSeq[T], rpos: BackwardsIndex): var T = + return s[int(s.len) - int(rpos)] + +func len*[T](s: ReservedMemSeq[T]): int = + s.mem.len div sizeof(T) + +proc setLen*[T](s: var ReservedMemSeq[T], newLen: int) = + # TODO call destructors + s.mem.setLen(newLen * sizeof(T)) + +proc add*[T](s: var ReservedMemSeq[T], val: T) = + let len = s.len + s.setLen(len + 1) + s[len] = val + +proc pop*[T](s: var ReservedMemSeq[T]): T = + assert s.usedMemEnd != s.memStart + let lastIdx = s.len - 1 + result = s[lastIdx] + s.setLen(lastIdx) + +func commitedLen*[T](s: ReservedMemSeq[T]): int = + s.mem.commitedLen div sizeof(T) + +func maxLen*[T](s: ReservedMemSeq[T]): int = + s.mem.maxLen div sizeof(T) + diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim index f13eb5e8e..8623a43e0 100644 --- a/lib/pure/strformat.nim +++ b/lib/pure/strformat.nim @@ -285,7 +285,11 @@ macro `&`*(pattern: string): untyped = var i = 0 let res = genSym(nskVar, "fmtRes") result = newNimNode(nnkStmtListExpr, lineInfoFrom=pattern) - result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + count(f, '{')*10))) + # XXX: https://github.com/nim-lang/Nim/issues/8405 + # When compiling with -d:useNimRtl, certain procs such as `count` from the strutils + # module are not accessible at compile-time: + let expectedGrowth = when defined(useNimRtl): 0 else: count(f, '{') * 10 + result.add newVarStmt(res, newCall(bindSym"newStringOfCap", newLit(f.len + expectedGrowth))) var strlit = "" while i < f.len: if f[i] == '{': diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 36e4e376f..2568f83c2 100644 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -143,51 +143,6 @@ template get(t: StringTableRef, key: string) = else: raise newException(KeyError, "key not found") -proc `[]=`*(t: StringTableRef, key, val: string) {. - rtlFunc, extern: "nstPut", noSideEffect.} - -proc newStringTable*(mode: StringTableMode): StringTableRef {. - rtlFunc, extern: "nst$1".} = - ## Creates a new empty string table. - ## - ## See also: - ## * `newStringTable(keyValuePairs) proc - ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ - new(result) - result.mode = mode - result.counter = 0 - newSeq(result.data, startSize) - -proc newStringTable*(keyValuePairs: varargs[string], - mode: StringTableMode): StringTableRef {. - rtlFunc, extern: "nst$1WithPairs".} = - ## Creates a new string table with given `key, value` string pairs. - ## - ## `StringTableMode` must be specified. - runnableExamples: - var mytab = newStringTable("key1", "val1", "key2", "val2", - modeCaseInsensitive) - - result = newStringTable(mode) - var i = 0 - while i < high(keyValuePairs): - result[keyValuePairs[i]] = keyValuePairs[i + 1] - inc(i, 2) - -proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], - mode: StringTableMode = modeCaseSensitive): StringTableRef {. - rtlFunc, extern: "nst$1WithTableConstr".} = - ## Creates a new string table with given `(key, value)` tuple pairs. - ## - ## The default mode is case sensitive. - runnableExamples: - var - mytab1 = newStringTable({"key1": "val1", "key2": "val2"}, modeCaseInsensitive) - mytab2 = newStringTable([("key3", "val3"), ("key4", "val4")]) - - result = newStringTable(mode) - for key, val in items(keyValuePairs): result[key] = val - proc len*(t: StringTableRef): int {.rtlFunc, extern: "nst$1".} = ## Returns the number of keys in `t`. result = t.counter @@ -292,6 +247,48 @@ proc `[]=`*(t: StringTableRef, key, val: string) {. rawInsert(t, t.data, key, val) inc(t.counter) +proc newStringTable*(mode: StringTableMode): StringTableRef {. + rtlFunc, extern: "nst$1".} = + ## Creates a new empty string table. + ## + ## See also: + ## * `newStringTable(keyValuePairs) proc + ## <#newStringTable,varargs[tuple[string,string]],StringTableMode>`_ + new(result) + result.mode = mode + result.counter = 0 + newSeq(result.data, startSize) + +proc newStringTable*(keyValuePairs: varargs[string], + mode: StringTableMode): StringTableRef {. + rtlFunc, extern: "nst$1WithPairs".} = + ## Creates a new string table with given `key, value` string pairs. + ## + ## `StringTableMode` must be specified. + runnableExamples: + var mytab = newStringTable("key1", "val1", "key2", "val2", + modeCaseInsensitive) + + result = newStringTable(mode) + var i = 0 + while i < high(keyValuePairs): + result[keyValuePairs[i]] = keyValuePairs[i + 1] + inc(i, 2) + +proc newStringTable*(keyValuePairs: varargs[tuple[key, val: string]], + mode: StringTableMode = modeCaseSensitive): StringTableRef {. + rtlFunc, extern: "nst$1WithTableConstr".} = + ## Creates a new string table with given `(key, value)` tuple pairs. + ## + ## The default mode is case sensitive. + runnableExamples: + var + mytab1 = newStringTable({"key1": "val1", "key2": "val2"}, modeCaseInsensitive) + mytab2 = newStringTable([("key3", "val3"), ("key4", "val4")]) + + result = newStringTable(mode) + for key, val in items(keyValuePairs): result[key] = val + proc raiseFormatException(s: string) = var e: ref ValueError new(e) diff --git a/lib/system.nim b/lib/system.nim index 441b7e263..eac13aeb3 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -417,6 +417,12 @@ when not defined(niminheritable): {.pragma: inheritable.} when not defined(nimunion): {.pragma: unchecked.} +when not defined(nimHasHotCodeReloading): + {.pragma: nonReloadable.} +when defined(hotCodeReloading): + {.pragma: hcrInline, inline.} +else: + {.pragma: hcrInline.} # comparison operators: proc `==`*[Enum: enum](x, y: Enum): bool {.magic: "EqEnum", noSideEffect.} @@ -1590,7 +1596,7 @@ when defined(nodejs) and not defined(nimscript): var programResult* {.importc: "process.exitCode".}: int programResult = 0 else: - var programResult* {.exportc: "nim_program_result".}: int + var programResult* {.compilerproc, exportc: "nim_program_result".}: int ## modify this variable to specify the exit code of the program ## under normal circumstances. When the program is terminated ## prematurely using ``quit``, this value is ignored. @@ -3802,6 +3808,15 @@ template doAssert*(cond: untyped, msg = "") = const expr = astToStr(cond) assertImpl(cond, msg, expr, true) +when compileOption("rangechecks"): + template rangeCheck*(cond) = + ## Helper for performing user-defined range checks. + ## Such checks will be performed only when the ``rangechecks`` + ## compile-time option is enabled. + if not cond: sysFatal(RangeError, "range check failed") +else: + template rangeCheck*(cond) = discard + iterator items*[T](a: seq[T]): T {.inline.} = ## iterates over each item of `a`. var i = 0 @@ -4251,3 +4266,6 @@ export widestrs import system/io export io + +when not defined(createNimHcr): + include nimhcr diff --git a/lib/system/cgprocs.nim b/lib/system/cgprocs.nim index 72219c2b7..9d0d248c3 100644 --- a/lib/system/cgprocs.nim +++ b/lib/system/cgprocs.nim @@ -13,8 +13,8 @@ type LibHandle = pointer # private type ProcAddr = pointer # library loading and loading of procs: -proc nimLoadLibrary(path: string): LibHandle {.compilerproc.} -proc nimUnloadLibrary(lib: LibHandle) {.compilerproc.} -proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc.} +proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} +proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} +proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} -proc nimLoadLibraryError(path: string) {.compilerproc, noinline.} +proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} diff --git a/lib/system/chcks.nim b/lib/system/chcks.nim index 0840d863a..789d709f7 100644 --- a/lib/system/chcks.nim +++ b/lib/system/chcks.nim @@ -8,7 +8,7 @@ # # Implementation of some runtime checks. -import system/indexerrors +include system/indexerrors proc raiseRangeError(val: BiggestInt) {.compilerproc, noinline.} = when hostOS == "standalone": diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 528587d05..74bdd5372 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -33,7 +33,7 @@ proc nimLoadLibraryError(path: string) = discard MessageBoxA(0, msg[0].addr, nil, 0) quit(1) -proc procAddrError(name: cstring) {.noinline.} = +proc procAddrError(name: cstring) {.compilerproc, nonReloadable, hcrInline.} = # carefully written to avoid memory allocation: cstderr.rawWrite("could not import: ") cstderr.rawWrite(name) diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 93fd693e0..cb2cda214 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -20,9 +20,9 @@ var proc c_fwrite(buf: pointer, size, n: csize, f: CFilePtr): cint {. importc: "fwrite", header: "<stdio.h>".} -proc rawWrite(f: CFilePtr, s: string|cstring) = +proc rawWrite(f: CFilePtr, s: cstring) {.compilerproc, nonreloadable, hcrInline.} = # we cannot throw an exception here! - discard c_fwrite(cstring(s), 1, s.len, f) + discard c_fwrite(s, 1, s.len, f) when not defined(windows) or not defined(guiapp): proc writeToStdErr(msg: cstring) = rawWrite(cstderr, msg) diff --git a/lib/system/gc_common.nim b/lib/system/gc_common.nim index 5af64ae20..91c0244ea 100644 --- a/lib/system/gc_common.nim +++ b/lib/system/gc_common.nim @@ -446,10 +446,10 @@ proc deallocHeap*(runFinalizers = true; allowGcAfterwards = true) = type GlobalMarkerProc = proc () {.nimcall, benign.} var - globalMarkersLen: int - globalMarkers: array[0..3499, GlobalMarkerProc] - threadLocalMarkersLen: int - threadLocalMarkers: array[0..3499, GlobalMarkerProc] + globalMarkersLen {.exportc.}: int + globalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] + threadLocalMarkersLen {.exportc.}: int + threadLocalMarkers {.exportc.}: array[0..3499, GlobalMarkerProc] gHeapidGenerator: int proc nimRegisterGlobalMarker(markerProc: GlobalMarkerProc) {.compilerProc.} = diff --git a/lib/system/memory.nim b/lib/system/memory.nim index f86fd4696..318dffa2d 100644 --- a/lib/system/memory.nim +++ b/lib/system/memory.nim @@ -11,7 +11,7 @@ proc nimCopyMem(dest, source: pointer, size: Natural) {.compilerproc, inline.} = d[i] = s[i] inc i -proc nimSetMem(a: pointer, v: cint, size: Natural) {.inline.} = +proc nimSetMem(a: pointer, v: cint, size: Natural) {.nonReloadable, inline.} = when useLibC: c_memset(a, v, size) else: @@ -22,7 +22,7 @@ proc nimSetMem(a: pointer, v: cint, size: Natural) {.inline.} = a[i] = v inc i -proc nimZeroMem(p: pointer, size: Natural) {.compilerproc, inline.} = +proc nimZeroMem(p: pointer, size: Natural) {.compilerproc, nonReloadable, inline.} = nimSetMem(p, 0, size) proc nimCmpMem(a, b: pointer, size: Natural): cint {.compilerproc, inline.} = @@ -37,7 +37,7 @@ proc nimCmpMem(a, b: pointer, size: Natural): cint {.compilerproc, inline.} = if d != 0: return d inc i -proc nimCStrLen(a: cstring): csize {.compilerproc, inline.} = +proc nimCStrLen(a: cstring): csize {.compilerproc, nonReloadable, inline.} = when useLibC: c_strlen(a) else: diff --git a/lib/system/sysstr.nim b/lib/system/sysstr.nim index 33cd4415f..9ba432459 100644 --- a/lib/system/sysstr.nim +++ b/lib/system/sysstr.nim @@ -82,7 +82,7 @@ proc copyStr(s: NimString, start: int): NimString {.compilerProc.} = if s == nil: return nil result = copyStrLast(s, start, s.len-1) -proc nimToCStringConv(s: NimString): cstring {.compilerProc, inline.} = +proc nimToCStringConv(s: NimString): cstring {.compilerProc, nonReloadable, inline.} = if s == nil or s.len == 0: result = cstring"" else: result = cstring(addr s.data) diff --git a/lib/windows/winlean.nim b/lib/windows/winlean.nim index d1bfbd447..49d65c251 100644 --- a/lib/windows/winlean.nim +++ b/lib/windows/winlean.nim @@ -683,6 +683,10 @@ const FILE_BEGIN* = 0'i32 INVALID_SET_FILE_POINTER* = -1'i32 NO_ERROR* = 0'i32 + PAGE_NOACCESS* = 0x01'i32 + PAGE_EXECUTE* = 0x10'i32 + PAGE_EXECUTE_READ* = 0x20'i32 + PAGE_EXECUTE_READWRITE* = 0x40'i32 PAGE_READONLY* = 2'i32 PAGE_READWRITE* = 4'i32 FILE_MAP_READ* = 4'i32 diff --git a/nimpretty/nimpretty.nim b/nimpretty/nimpretty.nim index c6c558f92..fc7eeba40 100644 --- a/nimpretty/nimpretty.nim +++ b/nimpretty/nimpretty.nim @@ -48,7 +48,7 @@ type proc prettyPrint(infile, outfile: string, opt: PrettyOptions) = var conf = newConfigRef() let fileIdx = fileInfoIdx(conf, AbsoluteFile infile) - conf.outFile = AbsoluteFile outfile + conf.outFile = RelativeFile outfile when defined(nimpretty2): var p: TParsers p.parser.em.indWidth = opt.indWidth diff --git a/testament/categories.nim b/testament/categories.nim index 337d15f4f..1d6c342a1 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -129,32 +129,36 @@ proc runBasicDLLTest(c, r: var TResults, cat: Category, options: string) = else: "" - var test1 = makeTest("lib/nimrtl.nim", options & " --app:lib -d:createNimRtl --threads:on", cat) + var test1 = makeTest("lib/nimrtl.nim", options & " --outdir:tests/dll", cat) test1.spec.action = actionCompile testSpec c, test1 - var test2 = makeTest("tests/dll/server.nim", options & " --app:lib -d:useNimRtl --threads:on" & rpath, cat) + var test2 = makeTest("tests/dll/server.nim", options & " --threads:on" & rpath, cat) test2.spec.action = actionCompile testSpec c, test2 + var test3 = makeTest("lib/nimhcr.nim", options & " --outdir:tests/dll" & rpath, cat) + test3.spec.action = actionCompile + testSpec c, test3 - when defined(Windows): - # windows looks in the dir of the exe (yay!): - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - else: + # windows looks in the dir of the exe (yay!): + when not defined(Windows): # posix relies on crappy LD_LIBRARY_PATH (ugh!): - const libpathenv = when defined(haiku): - "LIBRARY_PATH" - else: - "LD_LIBRARY_PATH" + const libpathenv = when defined(haiku): "LIBRARY_PATH" + else: "LD_LIBRARY_PATH" var libpath = getEnv(libpathenv).string # Temporarily add the lib directory to LD_LIBRARY_PATH: putEnv(libpathenv, "tests/dll" & (if libpath.len > 0: ":" & libpath else: "")) defer: putEnv(libpathenv, libpath) - var nimrtlDll = DynlibFormat % "nimrtl" - safeCopyFile("lib" / nimrtlDll, "tests/dll" / nimrtlDll) - testSpec r, makeTest("tests/dll/client.nim", options & " -d:useNimRtl --threads:on" & rpath, - cat) + testSpec r, makeTest("tests/dll/client.nim", options & " --threads:on" & rpath, cat) + testSpec r, makeTest("tests/dll/nimhcr_unit.nim", options & rpath, cat) + + if "boehm" notin options: + # force build required - see the comments in the .nim file for more details + var hcr_integration = makeTest("tests/dll/nimhcr_integration.nim", + options & " --forceBuild --hotCodeReloading:on" & rpath, cat) + hcr_integration.args = prepareTestArgs(hcr_integration.spec.getCmd, hcr_integration.name, + hcr_integration.options, getTestSpecTarget()) + testSpec r, hcr_integration proc dllTests(r: var TResults, cat: Category, options: string) = # dummy compile result: @@ -651,17 +655,17 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "", onStdout = if verboseMegatest: onStdout else: nil) if exitCode != 0: - echo buf + echo buf.string quit("megatest compilation failed") # Could also use onStdout here. (buf, exitCode) = execCmdEx("./megatest") if exitCode != 0: - echo buf + echo buf.string quit("megatest execution failed") - norm buf - writeFile("outputGotten.txt", buf) + norm buf.string + writeFile("outputGotten.txt", buf.string) var outputExpected = "" for i, runSpec in specs: outputExpected.add marker & runSpec.file & "\n" @@ -669,7 +673,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) = outputExpected.add '\n' norm outputExpected - if buf != outputExpected: + if buf.string != outputExpected: writeFile("outputExpected.txt", outputExpected) discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt") echo "output different!" diff --git a/testament/tester.nim b/testament/tester.nim index 9b3091fd5..1df08aa34 100644 --- a/testament/tester.nim +++ b/testament/tester.nim @@ -12,7 +12,9 @@ import parseutils, strutils, pegs, os, osproc, streams, parsecfg, json, marshal, backend, parseopt, specs, htmlgen, browsers, terminal, - algorithm, compiler/nodejs, times, sets, md5 + algorithm, times, sets, md5, sequtils + +include compiler/nodejs var useColors = true var backendLogging = true @@ -58,6 +60,7 @@ type name: string cat: Category options: string + args: seq[string] spec: TSpec startTime: float @@ -127,13 +130,17 @@ proc nimcacheDir(filename, options: string, target: TTarget): string = let hashInput = options & $target return "nimcache" / (filename & '_' & hashInput.getMD5) -proc callCompiler(cmdTemplate, filename, options: string, - target: TTarget, extraOptions=""): TSpec = +proc prepareTestArgs(cmdTemplate, filename, options: string, + target: TTarget, extraOptions=""): seq[string] = let nimcache = nimcacheDir(filename, options, target) let options = options & " " & quoteShell("--nimCache:" & nimcache) & extraOptions - let c = parseCmdLine(cmdTemplate % ["target", targetToCmd[target], - "options", options, "file", filename.quoteShell, - "filedir", filename.getFileDir()]) + return parseCmdLine(cmdTemplate % ["target", targetToCmd[target], + "options", options, "file", filename.quoteShell, + "filedir", filename.getFileDir()]) + +proc callCompiler(cmdTemplate, filename, options: string, + target: TTarget, extraOptions=""): TSpec = + let c = prepareTestArgs(cmdTemplate, filename, options, target, extraOptions) var p = startProcess(command=c[0], args=c[1 .. ^1], options={poStdErrToStdOut, poUsePath}) let outp = p.outputStream @@ -307,10 +314,13 @@ proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarg inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = - let (_, name, _) = test.name.splitFile - let ext = targetToExt[target] - result = nimcacheDir(test.name, test.options, target) / - ((if target == targetJS: "" else: "compiler_") & name.changeFileExt(ext)) + if target == targetJS: + result = test.name.changeFileExt("js") + else: + let (_, name, _) = test.name.splitFile + let ext = targetToExt[target] + result = nimcacheDir(test.name, test.options, target) / + ("compiler_" & name.changeFileExt(ext)) proc needsCodegenCheck(spec: TSpec): bool = result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0 @@ -368,6 +378,12 @@ proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, if given.err == reSuccess: inc(r.passed) r.addResult(test, target, expectedmsg, givenmsg, given.err) +proc getTestSpecTarget(): TTarget = + if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": + return targetCpp + else: + return targetC + proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = var expected = test.spec if expected.parseErrors.len > 0: @@ -384,10 +400,7 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = expected.targets.incl targets # still no target specified at all if expected.targets == {}: - if getEnv("NIM_COMPILE_TO_CPP", "false").string == "true": - expected.targets = {targetCpp} - else: - expected.targets = {targetC} + expected.targets = {getTestSpecTarget()} for target in expected.targets: inc(r.total) if target notin gTargets: @@ -409,21 +422,15 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = of actionRun: # In this branch of code "early return" pattern is clearer than deep # nested conditionals - the empty rows in between to clarify the "danger" - var given = callCompiler(expected.getCmd, test.name, test.options, - target) + var given = callCompiler(expected.getCmd, test.name, test.options, target) if given.err != reSuccess: r.addResult(test, target, "", given.msg, given.err) continue let isJsTarget = target == targetJS - var exeFile: string - if isJsTarget: - let file = test.name.lastPathPart.changeFileExt("js") - exeFile = nimcacheDir(test.name, test.options, target) / file - else: - exeFile = changeFileExt(test.name, ExeExt) - + var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) if not existsFile(exeFile): - r.addResult(test, target, expected.output, "executable not found", reExeNotFound) + r.addResult(test, target, expected.output, + "executable not found: " & exeFile, reExeNotFound) continue let nodejs = if isJsTarget: findNodeJs() else: "" @@ -432,10 +439,10 @@ proc testSpec(r: var TResults, test: TTest, targets: set[TTarget] = {}) = reExeNotFound) continue var exeCmd: string - var args: seq[string] + var args = test.args if isJsTarget: exeCmd = nodejs - args.add exeFile + args = concat(@[exeFile], args) else: exeCmd = exeFile var (buf, exitCode) = execCmdEx2(exeCmd, args, options = {poStdErrToStdOut}, input = expected.input) diff --git a/tests/dll/nimhcr_0.nim b/tests/dll/nimhcr_0.nim new file mode 100644 index 000000000..fe0b29a36 --- /dev/null +++ b/tests/dll/nimhcr_0.nim @@ -0,0 +1,4 @@ + +let g_0 = 1000 # new value! but also a "new" global :) + +proc getInt*(): int = return g_0 diff --git a/tests/dll/nimhcr_0_1.nim b/tests/dll/nimhcr_0_1.nim new file mode 100644 index 000000000..620050be3 --- /dev/null +++ b/tests/dll/nimhcr_0_1.nim @@ -0,0 +1,14 @@ + +import hotcodereloading + +let g_0 = 42 # lets start with the ultimate answer + +proc getInt*(): int = return g_0 + +programResult = 0 # should be accessible + +beforeCodeReload: + echo " 0: before" +afterCodeReload: + echo " 0: after" + \ No newline at end of file diff --git a/tests/dll/nimhcr_0_2.nim b/tests/dll/nimhcr_0_2.nim new file mode 100644 index 000000000..9ce228dc1 --- /dev/null +++ b/tests/dll/nimhcr_0_2.nim @@ -0,0 +1,18 @@ + +import hotcodereloading + +import nimhcr_1 # new import! + +# global scope for this module was executed when loading the program +# with a previous version which didn't contain this print statement +echo " 0: I SHOULDN'T BE PRINTED!" + +var g_0 = 0 # changed value but won't take effect + +proc getInt*(): int = return g_0 + g_1 + f_1() + +beforeCodeReload: + echo " 0: before - improved!" # changed handlers! +afterCodeReload: + echo " 0: after - improved!" + g_0 = 100 # we cannot change it in its initialization but we can in the 'after' handler! diff --git a/tests/dll/nimhcr_0_3.nim b/tests/dll/nimhcr_0_3.nim new file mode 100644 index 000000000..56f66e08c --- /dev/null +++ b/tests/dll/nimhcr_0_3.nim @@ -0,0 +1,18 @@ + +import hotcodereloading + +import nimhcr_1 +import nimhcr_2 # a new and different import! + +proc makeCounter*(): auto = + return iterator: int {.closure.} = + for i in countup(0, 10, 1): + yield i + +let c = makeCounter() + +afterCodeReload: + echo " 0: after - closure iterator: ", c() + echo " 0: after - closure iterator: ", c() + +proc getInt*(): int = return g_1 + g_2.len diff --git a/tests/dll/nimhcr_0_4.nim b/tests/dll/nimhcr_0_4.nim new file mode 100644 index 000000000..4471782a7 --- /dev/null +++ b/tests/dll/nimhcr_0_4.nim @@ -0,0 +1,19 @@ + +import hotcodereloading + +import nimhcr_1 # only importing 1 + +let g_0 = 1000 # new value! but also a "new" global :) + +proc getInt*(): int = return g_0 + +proc makeCounter*(): auto = + return iterator: int {.closure.} = + for i in countup(0, 10, 1): + yield i + +let c = makeCounter() + +afterCodeReload: + echo " 0: after - closure iterator! after reload! does it remember? :", c() + echo " 0: after - closure iterator! after reload! does it remember? :", c() diff --git a/tests/dll/nimhcr_0_5.nim b/tests/dll/nimhcr_0_5.nim new file mode 100644 index 000000000..aff4014ca --- /dev/null +++ b/tests/dll/nimhcr_0_5.nim @@ -0,0 +1,2 @@ + +proc getInt*(): int = return 42 # back to the answer... diff --git a/tests/dll/nimhcr_0_6.nim b/tests/dll/nimhcr_0_6.nim new file mode 100644 index 000000000..fe0b29a36 --- /dev/null +++ b/tests/dll/nimhcr_0_6.nim @@ -0,0 +1,4 @@ + +let g_0 = 1000 # new value! but also a "new" global :) + +proc getInt*(): int = return g_0 diff --git a/tests/dll/nimhcr_1.nim b/tests/dll/nimhcr_1.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/dll/nimhcr_1.nim diff --git a/tests/dll/nimhcr_1_1.nim b/tests/dll/nimhcr_1_1.nim new file mode 100644 index 000000000..299c52baa --- /dev/null +++ b/tests/dll/nimhcr_1_1.nim @@ -0,0 +1,51 @@ + +echo " 1: print me once!" + +import hotcodereloading + +let g_1* = 8 # devilish! + +proc f_1*(): int = + var a {.global.} = 1 + a.inc + return a + + +# all these constructs should compile +let some_glob_1 = 1 +echo " 1: ", some_glob_1 +if true: + let some_glob_2 = 2 + echo " 1: ", some_glob_2 + if true: + let some_glob_3 = 3 + echo " 1: ", some_glob_3 +block: + let some_glob_4 = 4 + proc inBlock(num: int) = + echo " 1: ", num + inBlock(some_glob_4) +var counter = 3 +while counter > 0: + let some_glob_5 = 5 + echo " 1: ", some_glob_5 + counter.dec + +type + Type1 = object + a: int + b: int +var t = Type1(a: 42, b: 11) +echo " 1: Type1.a:", t.a + +type + obj = ref object + dat: int + str: string + +proc foo(): (int, obj) = (1, obj(dat: 3, str: "bar")) + +let (aa, bb) = foo() +afterCodeReload: + echo aa + echo bb.str diff --git a/tests/dll/nimhcr_1_2.nim b/tests/dll/nimhcr_1_2.nim new file mode 100644 index 000000000..caf772450 --- /dev/null +++ b/tests/dll/nimhcr_1_2.nim @@ -0,0 +1,4 @@ + +import nimhcr_2 + +proc f_1*(): int = return f_2() diff --git a/tests/dll/nimhcr_1_3.nim b/tests/dll/nimhcr_1_3.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/dll/nimhcr_1_3.nim diff --git a/tests/dll/nimhcr_2.nim b/tests/dll/nimhcr_2.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/dll/nimhcr_2.nim diff --git a/tests/dll/nimhcr_2_1.nim b/tests/dll/nimhcr_2_1.nim new file mode 100644 index 000000000..a13b4c681 --- /dev/null +++ b/tests/dll/nimhcr_2_1.nim @@ -0,0 +1,15 @@ + +import hotcodereloading + +type + Type2 = ref object of RootObj + data*: int + +let g_2* = @[Type2(data: 2), Type2(data: 3)][1..^1] # should have a length of 1 + +var a: tuple[str: string, i: int] +a.str = " 2: random string" +echo a.str + +beforeCodeReload: + echo " 2: before!" diff --git a/tests/dll/nimhcr_2_2.nim b/tests/dll/nimhcr_2_2.nim new file mode 100644 index 000000000..04ab2d3bb --- /dev/null +++ b/tests/dll/nimhcr_2_2.nim @@ -0,0 +1,7 @@ + +import hotcodereloading + +proc f_2*(): int = return 1 + +afterCodeReload: + echo " 2: after!" diff --git a/tests/dll/nimhcr_2_3.nim b/tests/dll/nimhcr_2_3.nim new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/dll/nimhcr_2_3.nim diff --git a/tests/dll/nimhcr_integration.nim b/tests/dll/nimhcr_integration.nim new file mode 100644 index 000000000..daabe918f --- /dev/null +++ b/tests/dll/nimhcr_integration.nim @@ -0,0 +1,152 @@ +discard """ +output: ''' +main: HELLO! +main: hasAnyModuleChanged? true +main: before + 0: after +main: after + The answer is: 1000 +main: hasAnyModuleChanged? false + The answer is: 1000 +main: hasAnyModuleChanged? true + 0: before +main: before + 1: print me once! + 1: 1 + 1: 2 + 1: 3 + 1: 4 + 1: 5 + 1: 5 + 1: 5 + 1: Type1.a:42 +1 +bar + 0: after - improved! +main: after + The answer is: 110 +main: hasAnyModuleChanged? true + 0: before - improved! +main: before + 2: random string +1 +bar + 0: after - closure iterator: 0 + 0: after - closure iterator: 1 +main: after + The answer is: 9 +main: hasAnyModuleChanged? true + 2: before! +main: before + 2: after! + 0: after - closure iterator! after reload! does it remember? :2 + 0: after - closure iterator! after reload! does it remember? :3 +main: after + The answer is: 1000 +main: hasAnyModuleChanged? true +main: before +main: after + The answer is: 42 +done +''' +""" + +## This is perhaps the most complex test in the nim test suite - calling the +## compiler on the file itself with the same set or arguments and reloading +## parts of the program at runtime! In the same folder there are a few modules +## with names such as `nimhcr_<number>.nim`. Each of them has a few versions which +## are in the format of `nimhcr_<number>_<version>.nim`. The below code uses the +## `update` proc to say which of the modules should bump its version (and that +## is done by copying `nimhcr_<number>_<version>.nim` onto `nimhcr_<number>.nim`). +## The files should refer to each other (when importing) without the versions. +## A few files can be updated by calling `update` for each of their indexes +## and after that with a single call to `compileReloadExecute` the new version +## of the program will be compiled, reloaded, and the only thing the main module +## calls from `nimhcr_0.nim` (the procedure `getInt` proc) is called for a result. +## +## This test is expected to be executed with arguments - the full nim compiler +## command used for building it - so it can rebuild iself the same way - example: +## +## compiling: +## nim c --hotCodeReloading:on --nimCache:<folder> <this_file>.nim +## executing: +## <this_file>.exe nim c --hotCodeReloading:on --nimCache:<folder> <this_file>.nim + +import os, osproc, times, strutils, hotcodereloading + +import nimhcr_0 # getInt() - the only thing we continually call from the main module + +proc compileReloadExecute() = + # Remove the `--forceBuild` option - is there in the first place because: + # - when `koch test` is ran for the first time the nimcache is empty + # - when each of the variants are built (debug, release after that, different GCs) + # the main executable that gets built into the appropriate nimcache folder + # gets copied to the originally intended destination and is executed + # (this behaviour is only when the --hotCodeReloading option is used). + # - when `koch test` is ran again and the nimcache is full the executable files + # in the nimcache folder aren't relinked and therefore aren't copied to the + # originally intended destination - so when the binary at the intended + # destination is executed - it is actually a remnant from a previous execution. + # That is a problem because it points to shared objects to load from its own + # nimcache folder - the one used for building it - a previous run! And when + # this test changes other modules it references but the main module (this file) + # remains intact - the binary isn't replaced. `--forceBuild` fixes this but has + # to be applied only for the main build - the one done from koch, but when this + # binary triggers rebuilding itself here it shouldn't rebuild the main module - + # that would lead to replacing the main binary executable which is running! + let cmd = commandLineParams()[0..^1].join(" ").replace(" --forceBuild") + let (stdout, exitcode) = execCmdEx(cmd) + if exitcode != 0: + echo "COMPILATION ERROR!" + echo "COMMAND: ", cmd + echo "STDOUT: ", stdout + quit 1 + echo "main: hasAnyModuleChanged? ", hasAnyModuleChanged() + performCodeReload() + echo " The answer is: ", getInt() + +# there are 3 files and all of them start from their 1st version +var vers = [1, 1, 1] +proc update(file: int) = + proc getfile(mid: string): string = + let (path, _, _) = splitFile(currentSourcePath()) + return path & "/nimhcr_" & mid & ".nim" + copyFile(getfile($file & "_" & $vers[file]), getfile($file)) + inc vers[file] + +beforeCodeReload: + echo "main: before" + +afterCodeReload: + echo "main: after" + +echo "main: HELLO!" + +update 0 +compileReloadExecute() # versions are: 1 - - + +compileReloadExecute() # no change + +update 0 +update 1 +compileReloadExecute() # versions are: 2 1 - + +update 0 +update 2 +compileReloadExecute() # versions are: 3 1 1 + +update 0 +update 1 +update 2 +compileReloadExecute() # versions are: 4 2 2 + +update 0 +compileReloadExecute() # versions are: 5 2 2 + +# final update so there are no git modifications left after everything +# (the last versions are like the first files without a version suffix) +update 0 +update 1 +update 2 + +echo "done" diff --git a/tests/dll/nimhcr_unit.nim b/tests/dll/nimhcr_unit.nim new file mode 100644 index 000000000..f539a53c8 --- /dev/null +++ b/tests/dll/nimhcr_unit.nim @@ -0,0 +1,147 @@ +discard """ +output: ''' +fastcall_proc implementation #1 10 +11 +fastcall_proc implementation #2 20 +22 +fastcall_proc implementation #2 20 +22 +fastcall_proc implementation #3 30 +33 +fastcall_proc implementation #3 30 +33 +fastcall_proc implementation #3 30 +33 +fastcall_proc implementation #3 40 +43 +cdecl_proc implementation #1 10 +11 +cdecl_proc implementation #2 20 +22 +cdecl_proc implementation #2 20 +22 +cdecl_proc implementation #3 30 +33 +cdecl_proc implementation #3 30 +33 +cdecl_proc implementation #3 30 +33 +cdecl_proc implementation #3 40 +43 +stdcall_proc implementation #1 10 +11 +stdcall_proc implementation #2 20 +22 +stdcall_proc implementation #2 20 +22 +stdcall_proc implementation #3 30 +33 +stdcall_proc implementation #3 30 +33 +stdcall_proc implementation #3 30 +33 +stdcall_proc implementation #3 40 +43 +noconv_proc implementation #1 10 +11 +noconv_proc implementation #2 20 +22 +noconv_proc implementation #2 20 +22 +noconv_proc implementation #3 30 +33 +noconv_proc implementation #3 30 +33 +noconv_proc implementation #3 30 +33 +noconv_proc implementation #3 40 +43 +inline_proc implementation #1 10 +11 +inline_proc implementation #2 20 +22 +inline_proc implementation #2 20 +22 +inline_proc implementation #3 30 +33 +inline_proc implementation #3 30 +33 +inline_proc implementation #3 30 +33 +inline_proc implementation #3 40 +43 +''' +""" + +import macros + +macro carryOutTests(callingConv: untyped): untyped = + let + procName = $callingConv & "_proc" + globalName = $callingConv & "_global" + callingConv = callingConv + p1 = ident(procName & "1") + p2 = ident(procName & "2") + p3 = ident(procName & "3") + g1 = ident(globalName & "1") + g2 = ident(globalName & "2") + + result = quote do: + var `g1`: pointer = nil + if hcrRegisterGlobal("dummy_module", `globalName`, sizeof(int), nil, addr `g1`): + cast[ptr int](`g1`)[] = 10 + + var `g2`: pointer = nil + if hcrRegisterGlobal("dummy_module", `globalName`, sizeof(int), nil, addr `g2`): + cast[ptr int](`g2`)[] = 20 + + doAssert `g1` == `g2` and cast[ptr int](`g1`)[] == 10 + + type + F = proc (x: int): int {.placeholder.} + + proc `p1`(x: int): int {.placeholder.}= + echo `procName`, " implementation #1 ", x + return x + 1 + + let fp1 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p1`)) + echo fp1(10) + + proc `p2`(x: int): int {.placeholder.} = + echo `procName`, " implementation #2 ", x + return x + 2 + + let fp2 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p2`)) + echo fp1(20) + echo fp2(20) + + proc `p3`(x: int): int {.placeholder.} = + echo `procName`, " implementation #3 ", x + return x + 3 + + let fp3 = cast[F](hcrRegisterProc("dummy_module", `procName`, `p3`)) + echo fp1(30) + echo fp2(30) + echo fp3(30) + + let fp4 = cast[F](hcrGetProc("dummy_module", `procName`)) + echo fp4(40) + + proc replacePlaceholderPragmas(n: NimNode) = + if n.kind == nnkPragma: + n[0] = callingConv + else: + for i in 0 ..< n.len: + replacePlaceholderPragmas n[i] + + replacePlaceholderPragmas result + # echo result.treeRepr + +hcrAddModule("dummy_module") + +carryOutTests fastcall +carryOutTests cdecl +carryOutTests stdcall +carryOutTests noconv +carryOutTests inline + diff --git a/tests/dll/nimhcr_unit.nim.cfg b/tests/dll/nimhcr_unit.nim.cfg new file mode 100644 index 000000000..b13c310d9 --- /dev/null +++ b/tests/dll/nimhcr_unit.nim.cfg @@ -0,0 +1,2 @@ +-d:useNimRtl +-d:testNimHcr diff --git a/tests/dll/test_nimhcr_integration.bat b/tests/dll/test_nimhcr_integration.bat new file mode 100644 index 000000000..66e6beac4 --- /dev/null +++ b/tests/dll/test_nimhcr_integration.bat @@ -0,0 +1,10 @@ +set NIM=nim +set NIM_FLAGS=-d:debug + +%NIM% c --outdir:"." %NIM_FLAGS% ../../lib/nimrtl.nim +%NIM% c --outdir:"." %NIM_FLAGS% ../../lib/nimhcr.nim + +set HCR_FLAGS=--forceBuild --hotCodeReloading:on --nimcache:nimcache %NIM_FLAGS% + +%NIM% %HCR_FLAGS% c nimhcr_integration.nim +nimhcr_integration %NIM% %HCR_FLAGS% c nimhcr_integration.nim diff --git a/tests/dll/test_nimhcr_integration.sh b/tests/dll/test_nimhcr_integration.sh new file mode 100755 index 000000000..a2e2d0483 --- /dev/null +++ b/tests/dll/test_nimhcr_integration.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +rm -rf nimcache + +NIM_FLAGS=${*:- -d:debug} +NIM=nim + +$NIM c --outdir:"." $NIM_FLAGS ../../lib/nimrtl.nim +$NIM c --outdir:"." $NIM_FLAGS ../../lib/nimhcr.nim + +echo ===== Compiling HCR Integration Test ===== +HCR_FLAGS="--forceBuild --hotCodeReloading:on --nimcache:nimcache $NIM_FLAGS" +$NIM $HCR_FLAGS c nimhcr_integration.nim +export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH +./nimhcr_integration $NIM $HCR_FLAGS c nimhcr_integration.nim diff --git a/tests/types/tissues_types.nim b/tests/types/tissues_types.nim index d3643842e..e5039fa61 100644 --- a/tests/types/tissues_types.nim +++ b/tests/types/tissues_types.nim @@ -41,6 +41,13 @@ block t5648: var g = Foo() g.bar = 3 + var + mainPtr1: pointer = main + mainPtr2 = pointer(main) + mainPtr3 = cast[pointer](main) + + doAssert mainPtr1 == mainPtr2 and mainPtr2 == mainPtr3 + main() block t7581: diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index f109cbbab..e1b5c4271 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -356,7 +356,7 @@ proc buildDocs*(args: string) = let a = nimArgs & " " & args docHackJs = "dochack.js" - docHackJsSource = docHackDir / "nimcache" / docHackJs + docHackJsSource = docHackDir / docHackJs docHackJsDest = docHtmlOutput / docHackJs buildJS() # This call generates docHackJsSource let docup = webUploadOutput / NimVersion |