diff options
Diffstat (limited to 'compiler')
33 files changed, 845 insertions, 343 deletions
diff --git a/compiler/ast.nim b/compiler/ast.nim index e691cc175..607b497fb 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -227,7 +227,7 @@ type TNodeKinds* = set[TNodeKind] type - TSymFlag* = enum # already 33 flags! + TSymFlag* = enum # already 34 flags! sfUsed, # read access of sym (for warnings) or simply used sfExported, # symbol is exported from module sfFromGeneric, # symbol is instantiation of a generic; this is needed @@ -276,6 +276,8 @@ type # the calling side of the macro, not from the # implementation. sfGenSym # symbol is 'gensym'ed; do not add to symbol table + sfNonReloadable # symbol will be left as-is when hot code reloading is on - + # meaning that it won't be renamed and/or changed in any way TSymFlags* = set[TSymFlag] @@ -468,6 +470,7 @@ type nfDefaultParam # an automatically inserter default parameter nfDefaultRefsParam # a default param value references another parameter # the flag is applied to proc default values and to calls + nfExecuteOnReload # A top-level statement that will be executed during reloads TNodeFlags* = set[TNodeFlag] TTypeFlag* = enum # keep below 32 for efficiency reasons (now: beyond that) @@ -654,7 +657,7 @@ type mNIntVal, mNFloatVal, mNSymbol, mNIdent, mNGetType, mNStrVal, mNSetIntVal, mNSetFloatVal, mNSetSymbol, mNSetIdent, mNSetType, mNSetStrVal, mNLineInfo, - mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, + mNNewNimNode, mNCopyNimNode, mNCopyNimTree, mStrToIdent, mNSigHash, mNBindSym, mLocals, mNCallSite, mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym, mNHint, mNWarning, mNError, @@ -854,7 +857,7 @@ type offset*: int # offset of record field loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a - # reference to another object to safe space) + # reference to another object to save space) constraint*: PNode # additional constraints like 'lit|result'; also # misused for the codegenDecl pragma in the hope # it won't cause problems @@ -978,7 +981,8 @@ const PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfDotSetter, nfDotField, nfIsRef, nfPreventCg, nfLL, - nfFromTemplate, nfDefaultRefsParam} + nfFromTemplate, nfDefaultRefsParam, + nfExecuteOnReload} namePos* = 0 patternPos* = 1 # empty except for term rewriting macros genericParamsPos* = 2 diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index d177e1f88..4c8fa7147 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -185,7 +185,7 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) = var length = sonsLen(ri) for i in countup(1, length - 1): genParamLoop(params) - fixupCall(p, le, ri, d, op.r, params) + fixupCall(p, le, ri, d, rdLoc(op), params) proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = @@ -209,7 +209,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = genParamLoop(pl) template genCallPattern {.dirty.} = - lineF(p, cpsStmts, callPattern & ";$n", [op.r, pl, pl.addComma, rawProc]) + lineF(p, cpsStmts, callPattern & ";$n", [rdLoc(op), pl, pl.addComma, rawProc]) let rawProc = getRawProcType(p, typ) let callPattern = if tfIterator in typ.flags: PatIter else: PatProc @@ -237,7 +237,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = assert(d.t != nil) # generate an assignment to d: var list: TLoc initLoc(list, locCall, d.lode, OnUnknown) - list.r = callPattern % [op.r, pl, pl.addComma, rawProc] + list.r = callPattern % [rdLoc(op), pl, pl.addComma, rawProc] genAssignment(p, d, list, {}) # no need for deep copying else: genCallPattern() diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index f92f2e4de..a40c60e6d 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1534,6 +1534,7 @@ proc genDollar(p: BProc, n: PNode, d: var TLoc, frmt: string) = var a: TLoc initLocExpr(p, n.sons[1], a) a.r = ropecg(p.module, frmt, [rdLoc(a)]) + a.flags = a.flags - {lfIndirect} # this flag should not be propagated here (not just for HCR) if d.k == locNone: getTemp(p, n.typ, d) genAssignment(p, d, a, {}) gcUsage(p.config, n) @@ -2034,8 +2035,28 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) = genCall(p, e, d) of mNewString, mNewStringOfCap, mExit, mParseBiggestFloat: var opr = e.sons[0].sym + # Why would anyone want to set nodecl to one of these hardcoded magics? + # - not sure, and it wouldn't work if the symbol behind the magic isn't + # somehow forward-declared from some other usage, but it is *possible* if lfNoDecl notin opr.loc.flags: + let prc = magicsys.getCompilerProc(p.module.g.graph, $opr.loc.r) + # HACK: + # Explicitly add this proc as declared here so the cgsym call doesn't + # add a forward declaration - without this we could end up with the same + # 2 forward declarations. That happens because the magic symbol and the original + # one that shall be used have different ids (even though a call to one is + # actually a call to the other) so checking into m.declaredProtos with the 2 different ids doesn't work. + # Why would 2 identical forward declarations be a problem? + # - in the case of hot code-reloading we generate function pointers instead + # of forward declarations and in C++ it is an error to redefine a global + let wasDeclared = containsOrIncl(p.module.declaredProtos, prc.id) + # Make the function behind the magic get actually generated - this will + # not lead to a forward declaration! The genCall will lead to one. discard cgsym(p.module, $opr.loc.r) + # make sure we have pointer-initialising code for hot code reloading + if not wasDeclared and p.hcrOn: + addf(p.module.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [mangleDynLibProc(prc), getTypeDesc(p.module, prc.loc.t), getModuleDllPath(p.module, prc)]) genCall(p, e, d) of mReset: genReset(p, e) of mEcho: genEcho(p, e[1].skipConv) @@ -2292,6 +2313,7 @@ proc exprComplexConst(p: BProc, n: PNode, d: var TLoc) = proc expr(p: BProc, n: PNode, d: var TLoc) = p.currLineInfo = n.info + case n.kind of nkSym: var sym = n.sym diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index 2d68a198e..ccb5a7635 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -31,7 +31,7 @@ const cfsData: "NIM_merge_DATA", cfsProcs: "NIM_merge_PROCS", cfsInitProc: "NIM_merge_INIT_PROC", - cfsDatInitProc: "NIM_merge_DATINIT_PROC", + cfsDatInitProc: "NIM_merge_DATINIT_PROC", cfsTypeInit1: "NIM_merge_TYPE_INIT1", cfsTypeInit2: "NIM_merge_TYPE_INIT2", cfsTypeInit3: "NIM_merge_TYPE_INIT3", diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index bc8735397..6dc10db3d 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -15,18 +15,20 @@ const stringCaseThreshold = 8 # above X strings a hash-switch for strings is generated -proc registerGcRoot(p: BProc, v: PSym) = +proc getTraverseProc(p: BProc, v: Psym): Rope = if p.config.selectedGC in {gcMarkAndSweep, gcDestructors, gcV2, gcRefc} and containsGarbageCollectedRef(v.loc.t): # we register a specialized marked proc here; this has the advantage # that it works out of the box for thread local storage then :-) - let prc = genTraverseProcForGlobal(p.module, v, v.info) - if sfThread in v.flags: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterThreadLocalMarker($1);$n", [prc]) - else: - appcg(p.module, p.module.initProc.procSec(cpsInit), - "#nimRegisterGlobalMarker($1);$n", [prc]) + result = genTraverseProcForGlobal(p.module, v, v.info) + +proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) = + if sfThread in v.flags: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc]) + else: + appcg(p.module, p.module.initProc.procSec(cpsInit), + "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc]) proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} = if n.kind == nkEmpty: return false @@ -41,6 +43,10 @@ proc inExceptBlockLen(p: BProc): int = for x in p.nestedTryStmts: if x.inExcept: result.inc +proc startBlock(p: BProc, start: FormatStr = "{$n", + args: varargs[Rope]): int {.discardable.} +proc endBlock(p: BProc) + proc genVarTuple(p: BProc, n: PNode) = var tup, field: TLoc if n.kind != nkVarTuple: internalError(p.config, n.info, "genVarTuple") @@ -52,6 +58,32 @@ proc genVarTuple(p: BProc, n: PNode) = genStmts(p, lowerTupleUnpacking(p.module.g.graph, n, p.prc)) return + # check only the first son + var forHcr = treatGlobalDifferentlyForHCR(p.module, n.sons[0].sym) + let hcrCond = if forHcr: getTempName(p.module) else: nil + var hcrGlobals: seq[tuple[loc: TLoc, tp: Rope]] + # determine if the tuple is constructed at top-level scope or inside of a block (if/while/block) + let isGlobalInBlock = forHcr and p.blocks.len > 2 + # do not close and reopen blocks if this is a 'global' but inside of a block (if/while/block) + forHcr = forHcr and not isGlobalInBlock + + if forHcr: + # check with the boolean if the initializing code for the tuple should be ran + lineCg(p, cpsStmts, "if ($1)$n", hcrCond) + startBlock(p) + defer: + if forHcr: + # end the block where the tuple gets initialized + endBlock(p) + if forHcr or isGlobalInBlock: + # insert the registration of the globals for the different parts of the tuple at the + # start of the current scope (after they have been iterated) and init a boolean to + # check if any of them is newly introduced and the initializing code has to be ran + lineCg(p, cpsLocals, "NIM_BOOL $1 = NIM_FALSE;$n", hcrCond) + for curr in hcrGlobals: + lineCg(p, cpsLocals, "$1 |= hcrRegisterGlobal($4, \"$2\", sizeof($3), $5, (void**)&$2);$N", + hcrCond, curr.loc.r, rdLoc(curr.loc), getModuleDllPath(p.module, n.sons[0].sym), curr.tp) + genLineDir(p, n) initLocExpr(p, n.sons[L-1], tup) var t = tup.t.skipTypes(abstractInst) @@ -59,10 +91,13 @@ proc genVarTuple(p: BProc, n: PNode) = let vn = n.sons[i] let v = vn.sym if sfCompileTime in v.flags: continue + var traverseProc: Rope if sfGlobal in v.flags: assignGlobalVar(p, vn) genObjectInit(p, cpsInit, v.typ, v.loc, true) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: assignLocalVar(p, vn) initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[L-1])) @@ -73,6 +108,8 @@ proc genVarTuple(p: BProc, n: PNode) = if t.n.sons[i].kind != nkSym: internalError(p.config, n.info, "genVarTuple") field.r = "$1.$2" % [rdLoc(tup), mangleRecFieldName(p.module, t.n.sons[i].sym)] putLocIntoDest(p, v.loc, field) + if forHcr or isGlobalInBlock: + hcrGlobals.add((loc: v.loc, tp: if traverseProc == nil: ~"NULL" else: traverseProc)) proc genDeref(p: BProc, e: PNode, d: var TLoc; enforceDeref=false) @@ -242,6 +279,7 @@ proc genSingleVar(p: BProc, a: PNode) = genGotoVar(p, a.sons[2]) return var targetProc = p + var traverseProc: Rope if sfGlobal in v.flags: if v.flags * {sfImportc, sfExportc} == {sfImportc} and a.sons[2].kind == nkEmpty and @@ -270,7 +308,9 @@ proc genSingleVar(p: BProc, a: PNode) = # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc) if sfExportc in v.flags and p.module.g.generatedHeader != nil: genVarPrototype(p.module.g.generatedHeader, vn) - registerGcRoot(p, v) + traverseProc = getTraverseProc(p, v) + if traverseProc != nil and not p.hcrOn: + registerTraverseProc(p, v, traverseProc) else: let value = a.sons[2] let imm = isAssignedImmediately(p.config, value) @@ -302,6 +342,30 @@ proc genSingleVar(p: BProc, a: PNode) = assignLocalVar(p, vn) initLocalVar(p, v, imm) + if traverseProc == nil: traverseProc = ~"NULL" + # If the var is in a block (control flow like if/while or a block) in global scope just + # register the so called "global" so it can be used later on. There is no need to close + # and reopen of if (nim_hcr_do_init_) blocks because we are in one already anyway. + var forHcr = treatGlobalDifferentlyForHCR(p.module, v) + if forHcr and targetProc.blocks.len > 3 and v.owner.kind == skModule: + # put it in the locals section - mainly because of loops which + # use the var in a call to resetLoc() in the statements section + lineCg(targetProc, cpsLocals, "hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1);$n", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + # nothing special left to do later on - let's avoid closing and reopening blocks + forHcr = false + + # we close and reopen the global if (nim_hcr_do_init_) blocks in the main Init function + # for the module so we can have globals and top-level code be interleaved and still + # be able to re-run it but without the top level code - just the init of globals + if forHcr: + lineCg(targetProc, cpsStmts, "if (hcrRegisterGlobal($3, \"$1\", sizeof($2), $4, (void**)&$1))$N", + v.loc.r, rdLoc(v.loc), getModuleDllPath(p.module, v), traverseProc) + startBlock(targetProc) + defer: + if forHcr: + endBlock(targetProc) + if a.sons[2].kind != nkEmpty: genLineDir(targetProc, a) loadInto(targetProc, a.sons[0], a.sons[2], v.loc) diff --git a/compiler/ccgtrav.nim b/compiler/ccgtrav.nim index 0a2bbf93b..87e7c9d48 100644 --- a/compiler/ccgtrav.nim +++ b/compiler/ccgtrav.nim @@ -134,11 +134,13 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = var c: TTraversalClosure var p = newProc(nil, m) result = "Marker_" & getTypeName(m, origTyp, sig) - var typ = origTyp.skipTypes(abstractInstOwned) + let + hcrOn = m.hcrOn + typ = origTyp.skipTypes(abstractInstOwned) + markerName = if hcrOn: result & "_actual" else: result + header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [markerName] + t = getTypeDesc(m, typ) - let header = "static N_NIMCALL(void, $1)(void* p, NI op)" % [result] - - let t = getTypeDesc(m, typ) lineF(p, cpsLocals, "$1 a;$n", [t]) lineF(p, cpsInit, "a = ($1)p;$n", [t]) @@ -155,18 +157,23 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope = else: genTraverseProc(c, "(*a)".rope, typ.sons[0]) - let generatedProc = "$1 {$n$2$3$4}$n" % + let generatedProc = "$1 {$n$2$3$4}\n" % [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)] - m.s[cfsProcHeaders].addf("$1;$n", [header]) + m.s[cfsProcHeaders].addf("$1;\n", [header]) m.s[cfsProcs].add(generatedProc) + if hcrOn: + addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result]) + addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n", + [result, markerName, getModuleDllPath(m)]) + proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope = discard genTypeInfo(m, s.loc.t, info) var c: TTraversalClosure var p = newProc(nil, m) - var sLoc = s.loc.r + var sLoc = rdLoc(s.loc) result = getTempName(m) if sfThread in s.flags and emulatedThreadVars(m.config): diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index afe90544d..063c02df9 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -14,7 +14,7 @@ import sighashes from lowerings import createObj -proc genProcHeader(m: BModule, prc: PSym): Rope +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope proc isKeyword(w: PIdent): bool = # Nim and C++ share some keywords @@ -59,7 +59,23 @@ proc mangleParamName(m: BModule; s: PSym): Rope = result = s.loc.r if result == nil: var res = s.name.s.mangle - if isKeyword(s.name) or m.g.config.cppDefines.contains(res): + # Take into account if HCR is on because of the following scenario: + # if a module gets imported and it has some more importc symbols in it, + # some param names might recieve the "_0" suffix to distinguish from what + # is newly available. That might lead to changes in the C code in nimcache + # that contain only a parameter name change, but that is enough to mandate + # recompilation of that source file and thus a new shared object will be + # relinked. That may lead to a module getting reloaded which wasn't intended + # and that may be fatal when parts of the current active callstack when + # performCodeReload() was called are from the module being reloaded + # unintentionally - example (3 modules which import one another): + # main => proxy => reloadable + # we call performCodeReload() in proxy to reload only changes in reloadable + # but there is a new import which introduces an importc symbol `socket` + # and a function called in main or proxy uses `socket` as a parameter name. + # That would lead to either needing to reload `proxy` or to overwrite the + # executable file for the main module, which is running (or both!) -> error. + if m.hcrOn or isKeyword(s.name) or m.g.config.cppDefines.contains(res): res.add "_0" result = res.rope s.loc.r = result @@ -507,6 +523,8 @@ proc fillObjectFields*(m: BModule; typ: PType) = var check = initIntSet() discard getRecordFields(m, typ, check) +proc mangleDynLibProc(sym: PSym): Rope + proc getRecordDesc(m: BModule, typ: PType, name: Rope, check: var IntSet): Rope = # declare the record: @@ -535,22 +553,13 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope, appcg(m, result, " : public $1 {$n", [getTypeDescAux(m, typ.sons[0].skipTypes(skipPtrs), check)]) if typ.isException: - appcg(m, result, "virtual void raise() {throw *this;}$n") # required for polymorphic exceptions + appcg(m, result, "virtual void raise() { throw *this; }$n") # required for polymorphic exceptions if typ.sym.magic == mException: # Add cleanup destructor to Exception base class - appcg(m, result, "~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) - # hack: forward declare popCurrentExceptionEx() on top of type description, - # proper request to generate popCurrentExceptionEx not possible for 2 reasons: - # generated function will be below declared Exception type and circular dependency - # between Exception and popCurrentExceptionEx function - - let popExSym = magicsys.getCompilerProc(m.g.graph, "popCurrentExceptionEx") - if lfDynamicLib in popExSym.loc.flags and sfImportc in popExSym.flags: - # echo popExSym.flags, " ma flags ", popExSym.loc.flags - result = "extern " & getTypeDescAux(m, popExSym.typ, check) & " " & - mangleName(m, popExSym) & ";\L" & result - else: - result = genProcHeader(m, popExSym) & ";\L" & result + appcg(m, result, "~$1();$n", [name]) + # define it out of the class body and into the procs section so we don't have to + # artificially forward-declare popCurrentExceptionEx (very VERY troublesome for HCR) + appcg(m, cfsProcs, "inline $1::~$1() {if(this->raiseId) #popCurrentExceptionEx(this->raiseId);}$n", [name]) hasField = true else: appcg(m, result, " {$n $1 Sup;$n", @@ -888,31 +897,43 @@ proc finishTypeDescriptions(m: BModule) = template cgDeclFrmt*(s: PSym): string = s.constraint.strVal -proc genProcHeader(m: BModule, prc: PSym): Rope = +proc isReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable notin prc.flags + +proc isNonReloadable(m: BModule, prc: PSym): bool = + return m.hcrOn and sfNonReloadable in prc.flags + +proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope = var rettype, params: Rope - genCLineDir(result, prc.info, m.config) # using static is needed for inline procs if lfExportLib in prc.loc.flags: if isHeaderFile in m.flags: result.add "N_LIB_IMPORT " else: result.add "N_LIB_EXPORT " - elif prc.typ.callConv == ccInline: + elif prc.typ.callConv == ccInline or asPtr or isNonReloadable(m, prc): result.add "static " elif {sfImportc, sfExportc} * prc.flags == {}: result.add "N_LIB_PRIVATE " var check = initIntSet() fillLoc(prc.loc, locProc, prc.ast[namePos], mangleName(m, prc), OnUnknown) genProcParams(m, prc.typ, rettype, params, check) + # handle the 2 options for hotcodereloading codegen - function pointer + # (instead of forward declaration) or header for function budy with "_actual" postfix + let asPtrStr = rope(if asPtr: "_PTR" else: "") + var name = prc.loc.r + if isReloadable(m, prc) and not asPtr: + add(name, "_actual") # careful here! don't access ``prc.ast`` as that could reload large parts of # the object graph! if prc.constraint.isNil: - addf(result, "$1($2, $3)$4", - [rope(CallingConvToStr[prc.typ.callConv]), rettype, prc.loc.r, + addf(result, "$1$2($3, $4)$5", + [rope(CallingConvToStr[prc.typ.callConv]), asPtrStr, rettype, name, params]) else: - result = prc.cgDeclFrmt % [rettype, prc.loc.r, params] + let asPtrStr = if asPtr: (rope("(*") & name & ")") else: name + result = prc.cgDeclFrmt % [rettype, asPtrStr, params] # ------------------ type info generation ------------------------------------- @@ -921,6 +942,9 @@ proc getNimNode(m: BModule): Rope = result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)] inc(m.typeNodes) +proc TINameForHcr(m: BModule, name: Rope): Rope = + return if m.hcrOn: "(*".rope & name & ")" else: name + proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; name, base: Rope; info: TLineInfo) = var nimtypeKind: int @@ -930,19 +954,21 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; else: nimtypeKind = ord(typ.kind) + let nameHcr = TINameForHcr(m, name) + var size: Rope if tfIncompleteStruct in typ.flags: size = rope"void*" else: size = getTypeDesc(m, origType) addf(m.s[cfsTypeInit3], "$1.size = sizeof($2);$n" & "$1.kind = $3;$n" & "$1.base = $4;$n", - [name, size, rope(nimtypeKind), base]) + [nameHcr, size, rope(nimtypeKind), base]) # compute type flags for GC optimization var flags = 0 if not containsGarbageCollectedRef(typ): flags = flags or 1 if not canFormAcycle(typ): flags = flags or 2 #else MessageOut("can contain a cycle: " & typeToString(typ)) if flags != 0: - addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [name, rope(flags)]) + addf(m.s[cfsTypeInit3], "$1.flags = $2;$n", [nameHcr, rope(flags)]) discard cgsym(m, "TNimType") if isDefined(m.config, "nimTypeNames"): var typename = typeToString(if origType.typeInst != nil: origType.typeInst @@ -950,11 +976,17 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType; if typename == "ref object" and origType.skipTypes(skipPtrs).sym != nil: typename = "anon ref object from " & m.config$origType.skipTypes(skipPtrs).sym.info addf(m.s[cfsTypeInit3], "$1.name = $2;$n", - [name, makeCstring typename]) + [nameHcr, makeCstring typename]) discard cgsym(m, "nimTypeRoot") addf(m.s[cfsTypeInit3], "$1.nextType = nimTypeRoot; nimTypeRoot=&$1;$n", - [name]) - addf(m.s[cfsVars], "TNimType $1;$n", [name]) + [nameHcr]) + + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($2, \"$1\", sizeof(TNimType), NULL, (void**)&$1);$n", + [name, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsVars], "TNimType $1;$n", [name]) proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) = @@ -984,6 +1016,14 @@ proc discriminatorTableDecl(m: BModule, objtype: PType, d: PSym): Rope = var tmp = discriminatorTableName(m, objtype, d) result = "TNimNode* $1[$2];$n" % [tmp, rope(lengthOrd(m.config, d.typ)+1)] +proc genTNimNodeArray(m: BModule, name: Rope, size: Rope) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimNode** $1;$n", [name]) + addf(m.hcrCreateTypeInfosProc, "\thcrRegisterGlobal($3, \"$1\", sizeof(TNimNode*) * $2, NULL, (void**)&$1);$n", + [name, size, getModuleDllPath(m, m.module)]) + else: + addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [name, size]) + proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; info: TLineInfo) = case n.kind @@ -992,8 +1032,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope; if L == 1: genObjectFields(m, typ, origType, n.sons[0], expr, info) elif L > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(L)]) + var tmp = getTempName(m) & "_" & $L + genTNimNodeArray(m, tmp, rope(L)) for i in countup(0, L-1): var tmp2 = getNimNode(m) addf(m.s[cfsTypeInit3], "$1[$2] = &$3;$n", [tmp, rope(i), tmp2]) @@ -1066,7 +1106,7 @@ proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo var tmp = getNimNode(m) if not isImportedType(typ): genObjectFields(m, typ, origType, typ.n, tmp, info) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, tmp]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), tmp]) var t = typ.sons[0] while t != nil: t = t.skipTypes(skipPtrs) @@ -1078,8 +1118,8 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) var expr = getNimNode(m) var length = sonsLen(typ) if length > 0: - var tmp = getTempName(m) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", [tmp, rope(length)]) + var tmp = getTempName(m) & "_" & $length + genTNimNodeArray(m, tmp, rope(length)) for i in countup(0, length - 1): var a = typ.sons[i] var tmp2 = getNimNode(m) @@ -1094,7 +1134,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) else: addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2;$n", [expr, rope(length)]) - addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [name, expr]) + addf(m.s[cfsTypeInit3], "$1.node = &$2;$n", [TINameForHcr(m, name), expr]) proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # Type information for enumerations is quite heavy, so we do some @@ -1102,10 +1142,9 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = # anyway. We generate a cstring array and a loop over it. Exceptional # positions will be reset after the loop. genTypeInfoAux(m, typ, typ, name, info) - var nodePtrs = getTempName(m) var length = sonsLen(typ.n) - addf(m.s[cfsTypeInit1], "static TNimNode* $1[$2];$n", - [nodePtrs, rope(length)]) + var nodePtrs = getTempName(m) & "_" & $length + genTNimNodeArray(m, nodePtrs, rope(length)) var enumNames, specialCases: Rope var firstNimNode = m.typeNodes var hasHoles = false @@ -1134,17 +1173,17 @@ proc genEnumInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = add(m.s[cfsTypeInit3], specialCases) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n$4.node = &$1;$n", - [getNimNode(m), rope(length), nodePtrs, name]) + [getNimNode(m), rope(length), nodePtrs, TINameForHcr(m, name)]) if hasHoles: # 1 << 2 is {ntfEnumHole} - addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [name]) + addf(m.s[cfsTypeInit3], "$1.flags = 1<<2;$n", [TINameForHcr(m, name)]) proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = assert(typ.sons[0] != nil) genTypeInfoAux(m, typ, typ, name, info) var tmp = getNimNode(m) addf(m.s[cfsTypeInit3], "$1.len = $2; $1.kind = 0;$n" & "$3.node = &$1;$n", - [tmp, rope(firstOrd(m.config, typ)), name]) + [tmp, rope(firstOrd(m.config, typ)), TINameForHcr(m, name)]) proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) = genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ.sons[1], info), info) @@ -1169,19 +1208,29 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = let origType = t var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses) + let prefixTI = if m.hcrOn: "(" else: "(&" + let sig = hashType(origType) result = m.typeInfoMarker.getOrDefault(sig) if result != nil: - return "(&".rope & result & ")".rope + return prefixTI.rope & result & ")".rope - result = m.g.typeInfoMarker.getOrDefault(sig) - if result != nil: + proc declareNimType(m: BModule, str: Rope, ownerModule: PSym) = + if m.hcrOn: + addf(m.s[cfsVars], "static TNimType* $1;$n", [str]) + addf(m.s[cfsTypeInit1], "\t$1 = (TNimType*)hcrGetGlobal($2, \"$1\");$n", + [str, getModuleDllPath(m, ownerModule)]) + else: + addf(m.s[cfsVars], "extern TNimType $1;$n", [str]) + + let marker = m.g.typeInfoMarker.getOrDefault(sig) + if marker.str != nil: discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) + declareNimType(m, marker.str, marker.owner) # also store in local type section: - m.typeInfoMarker[sig] = result - return "(&".rope & result & ")".rope + m.typeInfoMarker[sig] = marker.str + return prefixTI.rope & marker.str & ")".rope result = "NTI$1_" % [rope($sig)] m.typeInfoMarker[sig] = result @@ -1194,10 +1243,10 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = # reference the type info as extern here discard cgsym(m, "TNimType") discard cgsym(m, "TNimNode") - addf(m.s[cfsVars], "extern TNimType $1;$n", [result]) - return "(&".rope & result & ")".rope + declareNimType(m, result, owner) + return prefixTI.rope & result & ")".rope - m.g.typeInfoMarker[sig] = result + m.g.typeInfoMarker[sig] = (str: result, owner: owner) case t.kind of tyEmpty, tyVoid: result = rope"0" of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent: @@ -1219,12 +1268,12 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = if m.config.selectedGC != gcDestructors: if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyRef: genTypeInfoAux(m, t, t, result, info) if m.config.selectedGC >= gcMarkAndSweep: let markerProc = genTraverseProc(m, origType, sig) - addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [result, markerProc]) + addf(m.s[cfsTypeInit3], "$1.marker = $2;$n", [TINameForHcr(m, result), markerProc]) of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info) of tyArray: genArrayInfo(m, t, result, info) of tySet: genSetInfo(m, t, result, info) @@ -1241,7 +1290,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope = genDeepCopyProc(m, t.deepCopy, result) elif origType.deepCopy != nil: genDeepCopyProc(m, origType.deepCopy, result) - result = "(&".rope & result & ")".rope + result = prefixTI.rope & result & ")".rope proc genTypeSection(m: BModule, n: PNode) = discard diff --git a/compiler/cgen.nim b/compiler/cgen.nim index cb186de10..d8f426f05 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,8 +16,6 @@ import condsyms, rodutils, renderer, idgen, cgendata, ccgmerge, semfold, aliases, lowerings, tables, sets, ndi, lineinfos, pathutils, transf -import system/indexerrors - when not defined(leanCompiler): import semparallel @@ -46,6 +44,9 @@ when options.hasTinyCBackend: # implementation +proc hcrOn(m: BModule): bool = m.config.hcrOn +proc hcrOn(p: BProc): bool = p.module.config.hcrOn + proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) @@ -92,6 +93,17 @@ proc useHeader(m: BModule, sym: PSym) = proc cgsym(m: BModule, name: string): Rope +proc getCFile(m: BModule): AbsoluteFile + +proc getModuleDllPath(m: BModule): Rope = + let (dir, name, ext) = splitFile(getCFile(m)) + let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext]) + return makeCString(dir.string & "/" & filename) + +proc getModuleDllPath(m: BModule, s: PSym): Rope = + return getModuleDllPath(findPendingModule(m, s)) + +# TODO: please document proc ropecg(m: BModule, frmt: FormatStr, args: varargs[Rope]): Rope = assert m != nil var i = 0 @@ -440,12 +452,18 @@ proc assignLocalVar(p: BProc, n: PNode) = include ccgthreadvars proc varInDynamicLib(m: BModule, sym: PSym) -proc mangleDynLibProc(sym: PSym): Rope + +proc treatGlobalDifferentlyForHCR(m: BModule, s: PSym): bool = + return m.hcrOn and {sfThread, sfGlobal} * s.flags == {sfGlobal} and + ({lfNoDecl, lfHeader} * s.loc.flags == {}) + # and s.owner.kind == skModule # owner isn't always a module (global pragma on local var) + # and s.loc.k == locGlobalVar # loc isn't always initialized when this proc is used proc assignGlobalVar(p: BProc, n: PNode) = let s = n.sym if s.loc.k == locNone: fillLoc(s.loc, locGlobalVar, n, mangleName(p.module, s), OnHeap) + if treatGlobalDifferentlyForHCR(p.module, s): incl(s.loc.flags, lfIndirect) if lfDynamicLib in s.loc.flags: var q = findPendingModule(p.module, s) @@ -463,8 +481,10 @@ proc assignGlobalVar(p: BProc, n: PNode) = var decl: Rope = nil var td = getTypeDesc(p.module, s.loc.t) if s.constraint.isNil: - if sfImportc in s.flags: add(decl, "extern ") + if p.hcrOn: add(decl, "static ") + elif sfImportc in s.flags: add(decl, "extern ") add(decl, td) + if p.hcrOn: add(decl, "*") if sfRegister in s.flags: add(decl, " register") if sfVolatile in s.flags: add(decl, " volatile") addf(decl, " $1;$n", [s.loc.r]) @@ -853,6 +873,14 @@ proc allPathsAsgnResult(n: PNode): InitResultEnum = for i in 0..<safeLen(n): allPathsInBranch(n[i]) +proc getProcTypeCast(m: BModule, prc: PSym): Rope = + result = getTypeDesc(m, prc.loc.t) + if prc.typ.callConv == ccClosure: + var rettype, params: Rope + var check = initIntSet() + genProcParams(m, prc.typ, rettype, params, check) + result = "$1(*)$2" % [rettype, params] + proc genProcAux(m: BModule, prc: PSym) = var p = newProc(prc, m) var header = genProcHeader(m, prc) @@ -931,6 +959,9 @@ proc genProcAux(m: BModule, prc: PSym) = add(generatedProc, returnStmt) add(generatedProc, ~"}$N") add(m.s[cfsProcs], generatedProc) + if isReloadable(m, prc): + addf(m.s[cfsDynLibInit], "\t$1 = ($3) hcrRegisterProc($4, \"$1\", (void*)$2);$n", + [prc.loc.r, prc.loc.r & "_actual", getProcTypeCast(m, prc), getModuleDllPath(m, prc)]) proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} = result = (sfCompileToCpp in m.module.flags and @@ -946,20 +977,27 @@ proc genProcPrototype(m: BModule, sym: PSym) = if lfDynamicLib in sym.loc.flags: if getModule(sym).id != m.module.id and not containsOrIncl(m.declaredThings, sym.id): - add(m.s[cfsVars], ropecg(m, "extern $1 $2;$n", + add(m.s[cfsVars], ropecg(m, "$1 $2 $3;$n", + rope(if isReloadable(m, sym): "static" else: "extern"), getTypeDesc(m, sym.loc.t), mangleDynLibProc(sym))) + if isReloadable(m, sym): + addf(m.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [mangleDynLibProc(sym), getTypeDesc(m, sym.loc.t), getModuleDllPath(m, sym)]) elif not containsOrIncl(m.declaredProtos, sym.id): - var header = genProcHeader(m, sym) - if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props: - header = "__declspec(noreturn) " & header - if sym.typ.callConv != ccInline and requiresExternC(m, sym): - header = "extern \"C\" " & header - if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props: - header.add(" __attribute__((naked))") - if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props: - header.add(" __attribute__((noreturn))") - add(m.s[cfsProcHeaders], ropecg(m, "$1;$n", header)) - + let asPtr = isReloadable(m, sym) + var header = genProcHeader(m, sym, asPtr) + if not asPtr: + if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props: + header = "__declspec(noreturn) " & header + if sym.typ.callConv != ccInline and requiresExternC(m, sym): + header = "extern \"C\" " & header + if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props: + header.add(" __attribute__((naked))") + if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props: + header.add(" __attribute__((noreturn))") + add(m.s[cfsProcHeaders], ropecg(m, "$1;$N", header)) + +# TODO: figure out how to rename this - it DOES generate a forward declaration proc genProcNoForward(m: BModule, prc: PSym) = if lfImportCompilerProc in prc.loc.flags: fillProcLoc(m, prc.ast[namePos]) @@ -969,7 +1007,6 @@ proc genProcNoForward(m: BModule, prc: PSym) = return if lfNoDecl in prc.loc.flags: fillProcLoc(m, prc.ast[namePos]) - useHeader(m, prc) genProcPrototype(m, prc) elif prc.typ.callConv == ccInline: # We add inline procs to the calling module to enable C based inlining. @@ -977,30 +1014,45 @@ proc genProcNoForward(m: BModule, prc: PSym) = # a check for ``m.declaredThings``. if not containsOrIncl(m.declaredThings, prc.id): #if prc.loc.k == locNone: - fillProcLoc(m, prc.ast[namePos]) + # mangle the inline proc based on the module where it is defined - not on the first module that uses it + fillProcLoc(findPendingModule(m, prc), prc.ast[namePos]) #elif {sfExportc, sfImportc} * prc.flags == {}: # # reset name to restore consistency in case of hashing collisions: # echo "resetting ", prc.id, " by ", m.module.name.s # prc.loc.r = nil # prc.loc.r = mangleName(m, prc) - useHeader(m, prc) genProcPrototype(m, prc) genProcAux(m, prc) elif lfDynamicLib in prc.loc.flags: var q = findPendingModule(m, prc) fillProcLoc(q, prc.ast[namePos]) - useHeader(m, prc) genProcPrototype(m, prc) if q != nil and not containsOrIncl(q.declaredThings, prc.id): symInDynamicLib(q, prc) + # register the procedure even though it is in a different dynamic library and will not be + # reloadable (and has no _actual suffix) - other modules will need to be able to get it through + # the hcr dynlib (also put it in the DynLibInit section - right after it gets loaded) + if isReloadable(q, prc): + addf(q.s[cfsDynLibInit], "\t$1 = ($2) hcrRegisterProc($3, \"$1\", (void*)$1);$n", + [prc.loc.r, getTypeDesc(q, prc.loc.t), getModuleDllPath(m, q.module)]) else: symInDynamicLibPartial(m, prc) elif sfImportc notin prc.flags: var q = findPendingModule(m, prc) fillProcLoc(q, prc.ast[namePos]) - useHeader(m, prc) + # generate a getProc call to initialize the pointer for this + # externally-to-the-current-module defined proc, also important + # to do the declaredProtos check before the call to genProcPrototype + if isReloadable(m, prc) and prc.id notin m.declaredProtos and + q != nil and q.module.id != m.module.id: + addf(m.s[cfsDynLibInit], "\t$1 = ($2) hcrGetProc($3, \"$1\");$n", + [prc.loc.r, getProcTypeCast(m, prc), getModuleDllPath(m, prc)]) genProcPrototype(m, prc) if q != nil and not containsOrIncl(q.declaredThings, prc.id): + # make sure there is a "prototype" in the external module + # which will actually become a function pointer + if isReloadable(m, prc): + genProcPrototype(q, prc) genProcAux(q, prc) else: fillProcLoc(m, prc.ast[namePos]) @@ -1049,6 +1101,8 @@ proc genVarPrototype(m: BModule, n: PNode) = let sym = n.sym useHeader(m, sym) fillLoc(sym.loc, locGlobalVar, n, mangleName(m, sym), OnHeap) + if treatGlobalDifferentlyForHCR(m, sym): incl(sym.loc.flags, lfIndirect) + if (lfNoDecl in sym.loc.flags) or contains(m.declaredThings, sym.id): return if sym.owner.id != m.module.id: @@ -1057,12 +1111,17 @@ proc genVarPrototype(m: BModule, n: PNode) = if sfThread in sym.flags: declareThreadVar(m, sym, true) else: - add(m.s[cfsVars], "extern ") + incl(m.declaredThings, sym.id) + add(m.s[cfsVars], if m.hcrOn: "static " else: "extern ") add(m.s[cfsVars], getTypeDesc(m, sym.loc.t)) + if m.hcrOn: add(m.s[cfsVars], "*") if lfDynamicLib in sym.loc.flags: add(m.s[cfsVars], "*") if sfRegister in sym.flags: add(m.s[cfsVars], " register") if sfVolatile in sym.flags: add(m.s[cfsVars], " volatile") addf(m.s[cfsVars], " $1;$n", [sym.loc.r]) + if m.hcrOn: addf(m.initProc.procSec(cpsLocals), + "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r, + getTypeDesc(m, sym.loc.t), getModuleDllPath(m, sym)]) const frameDefines = """ @@ -1106,22 +1165,76 @@ proc getCopyright(conf: ConfigRef; cfile: Cfile): Rope = proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope = result = getCopyright(conf, cfile) + if conf.hcrOn: add(result, "#define NIM_HOT_CODE_RELOADING\L") addIntTypes(result, conf) +proc getSomeNameForModule(m: PSym): Rope = + assert m.kind == skModule + assert m.owner.kind == skPackage + if {sfSystemModule, sfMainModule} * m.flags == {}: + result = m.owner.name.s.mangle.rope + result.add "_" + result.add m.name.s.mangle + +proc getSomeInitName(m: BModule, suffix: string): Rope = + if not m.hcrOn: + result = getSomeNameForModule(m.module) + result.add suffix + +proc getInitName(m: BModule): Rope = + if sfMainModule in m.module.flags: + # generate constant name for main module, for "easy" debugging. + result = rope"NimMainModule" + else: + result = getSomeInitName(m, "Init000") + +proc getDatInitName(m: BModule): Rope = getSomeInitName(m, "DatInit000") +proc getHcrInitName(m: BModule): Rope = getSomeInitName(m, "HcrInit000") + +proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): Rope + proc genMainProc(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed - const + var preMainCode: Rope + if m.hcrOn: + proc loadLib(handle: string, name: string): Rope = + let prc = magicsys.getCompilerProc(m.g.graph, name) + assert prc != nil + let n = newStrNode(nkStrLit, prc.annex.path.strVal) + n.info = prc.annex.path.info + appcg(m, result, "\tif (!($1 = #nimLoadLibrary($2)))$N" & + "\t\t#nimLoadLibraryError($2);$N", + [handle.rope, genStringLiteral(m, n)]) + + add(preMainCode, loadLib("hcr_handle", "hcrGetProc")) + add(preMainCode, "\tvoid* rtl_handle;$N") + add(preMainCode, loadLib("rtl_handle", "nimGC_setStackBottom")) + add(preMainCode, hcrGetProcLoadCode(m, "nimGC_setStackBottom", "nimrtl_", "rtl_handle", "nimGetProcAddr")) + add(preMainCode, "\tinner = PreMain;$N") + add(preMainCode, "\tinitStackBottomWith_actual((void *)&inner);$N") + add(preMainCode, "\t(*inner)();$N") + else: + add(preMainCode, "\tPreMain();$N") + + let + # not a big deal if we always compile these 3 global vars... makes the HCR code easier + PosixCmdLine = + "int cmdCount;$N" & + "char** cmdLine;$N" & + "char** gEnv;$N" + # The use of a volatile function pointer to call Pre/NimMainInner # prevents inlining of the NimMainInner function and dependent # functions, which might otherwise merge their stack frames. - PreMainBody = + PreMainBody = "$N" & "void PreMainInner(void) {$N" & "$2" & "$3" & "}$N$N" & + PosixCmdLine & "void PreMain(void) {$N" & "\tvoid (*volatile inner)(void);$N" & "\tinner = PreMainInner;$N" & @@ -1133,7 +1246,7 @@ proc genMainProc(m: BModule) = "\tNimMain();$N" MainProcsWithResult = - MainProcs & "\treturn nim_program_result;$N" + MainProcs & ("\treturn " & (if m.hcrOn: "*" else: "") & "nim_program_result;$N") NimMainInner = "N_CDECL(void, NimMainInner)(void) {$N" & "$1" & @@ -1142,7 +1255,7 @@ proc genMainProc(m: BModule) = NimMainProc = "N_CDECL(void, NimMain)(void) {$N" & "\tvoid (*volatile inner)(void);$N" & - "\tPreMain();$N" & + $preMainCode & "\tinner = NimMainInner;$N" & "$2" & "\t(*inner)();$N" & @@ -1150,12 +1263,6 @@ proc genMainProc(m: BModule) = NimMainBody = NimMainInner & NimMainProc - PosixNimMain = - "int cmdCount;$N" & - "char** cmdLine;$N" & - "char** gEnv;$N" & - NimMainBody - PosixCMain = "int main(int argc, char** args, char** env) {$N" & "\tcmdLine = args;$N" & @@ -1227,10 +1334,10 @@ proc genMainProc(m: BModule) = nimMain = PosixNimDllMain otherMain = PosixCDllMain elif m.config.target.targetOS == osStandalone: - nimMain = PosixNimMain + nimMain = NimMainBody otherMain = StandaloneCMain else: - nimMain = PosixNimMain + nimMain = NimMainBody otherMain = PosixCMain if optEndb in m.config.options: for i in 0..<m.config.m.fileInfos.len: @@ -1253,28 +1360,53 @@ proc genMainProc(m: BModule) = appcg(m, m.s[cfsProcs], otherMain, []) if m.config.cppCustomNamespace.len > 0: m.s[cfsProcs].add openNamespaceNim(m.config.cppCustomNamespace) -proc getSomeInitName(m: PSym, suffix: string): Rope = - assert m.kind == skModule - assert m.owner.kind == skPackage - if {sfSystemModule, sfMainModule} * m.flags == {}: - result = m.owner.name.s.mangle.rope - result.add "_" - result.add m.name.s.mangle - result.add suffix - -proc getInitName(m: PSym): Rope = - if sfMainModule in m.flags: - # generate constant name for main module, for "easy" debugging. - result = rope"NimMainModule" - else: - result = getSomeInitName(m, "Init000") - -proc getDatInitName(m: PSym): Rope = getSomeInitName(m, "DatInit000") - - proc registerModuleToMain(g: BModuleList; m: BModule) = + let + init = m.getInitName + datInit = m.getDatInitName + + if m.hcrOn: + var hcr_module_meta = "$nN_LIB_PRIVATE const char* hcr_module_list[] = {$n" % [] + let systemModulePath = getModuleDllPath(m, g.modules[g.graph.config.m.systemFileIdx.int].module) + let mainModulePath = getModuleDllPath(m, m.module) + if sfMainModule in m.module.flags: + addf(hcr_module_meta, "\t$1,$n", [systemModulePath]) + g.graph.importDeps.withValue(FileIndex(m.module.position), deps): + for curr in deps[]: + addf(hcr_module_meta, "\t$1,$n", [getModuleDllPath(m, g.modules[curr.int].module)]) + addf(hcr_module_meta, "\t\"\"};$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(void**, HcrGetImportedModules)() { return (void**)hcr_module_list; }$n", []) + addf(hcr_module_meta, "$nN_LIB_EXPORT N_NIMCALL(char*, HcrGetSigHash)() { return \"$1\"; }$n$n", + [($sigHash(m.module)).rope]) + if sfMainModule in m.module.flags: + add(g.mainModProcs, hcr_module_meta) + addf(g.mainModProcs, "static void* hcr_handle;$N", []) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [init]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void);$N", [datInit]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, $1)(void*, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*));$N", [m.getHcrInitName]) + addf(g.mainModProcs, "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void);$N", []) + addf(g.mainModInit, "\t$1();$N", [init]) + addf(g.otherModsInit, "\thcrInit((void**)hcr_module_list, $1, $2, $3, hcr_handle, nimGetProcAddr);$n", + [mainModulePath, systemModulePath, datInit]) + addf(g.mainDatInit, "\t$1(hcr_handle, nimGetProcAddr);$N", [m.getHcrInitName]) + addf(g.mainDatInit, "\thcrAddModule($1);\n", [mainModulePath]) + addf(g.mainDatInit, "\tHcrCreateTypeInfos();$N", []) + # nasty nasty hack to get the command line functionality working with HCR + # register the 2 variables on behalf of the os module which might not even + # be loaded (in which case it will get collected but that is not a problem) + let osModulePath = ($systemModulePath).replace("stdlib_system", "stdlib_os").rope + addf(g.mainDatInit, "\thcrAddModule($1);\n", [osModulePath]) + add(g.mainDatInit, "\tint* cmd_count;\n") + add(g.mainDatInit, "\tchar*** cmd_line;\n") + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdCount\", sizeof(cmd_count), NULL, (void**)&cmd_count);$N", [osModulePath]) + addf(g.mainDatInit, "\thcrRegisterGlobal($1, \"cmdLine\", sizeof(cmd_line), NULL, (void**)&cmd_line);$N", [osModulePath]) + add(g.mainDatInit, "\t*cmd_count = cmdCount;\n") + add(g.mainDatInit, "\t*cmd_line = cmdLine;\n") + else: + add(m.s[cfsInitProc], hcr_module_meta) + return + if m.s[cfsDatInitProc].len > 0: - let datInit = m.module.getDatInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit]) addf(g.mainDatInit, "\t$1();$N", [datInit]) @@ -1287,7 +1419,6 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = add(g.mainDatInit, ropecg(m, "\t#initStackBottomWith((void *)&inner);$N")) if m.s[cfsInitProc].len > 0: - let init = m.module.getInitName addf(g.mainModProcs, "N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init]) let initCall = "\t$1();$N" % [init] if sfMainModule in m.module.flags: @@ -1302,10 +1433,14 @@ proc genDatInitCode(m: BModule) = ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed - var moduleDatInitRequired = false + var moduleDatInitRequired = m.hcrOn + + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), getDatInitName(m)] - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % - [getDatInitName(m.module)] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) for i in cfsTypeInit1..cfsDynLibInit: if m.s[i].len != 0: @@ -1319,45 +1454,73 @@ proc genDatInitCode(m: BModule) = if moduleDatInitRequired: add(m.s[cfsDatInitProc], prc) +# Very similar to the contents of symInDynamicLib - basically only the +# things needed for the hot code reloading runtime procs to be loaded +proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): Rope = + let prc = magicsys.getCompilerProc(m.g.graph, sym) + assert prc != nil + fillProcLoc(m, prc.ast[namePos]) + + var extname = prefix & sym + var tmp = mangleDynLibProc(prc) + prc.loc.r = tmp + prc.typ.sym = nil + + if not containsOrIncl(m.declaredThings, prc.id): + addf(m.s[cfsVars], "static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t)]) + + result = "\t$1 = ($2) $3($4, $5);$n" % + [tmp, getTypeDesc(m, prc.typ), getProcFunc.rope, handle.rope, makeCString(prefix & sym)] + proc genInitCode(m: BModule) = ## this function is called in cgenWriteModules after all modules are closed, ## it means raising dependency on the symbols is too late as it will not propogate ## into other modules, only simple rope manipulations are allowed appcg(m, m.s[cfsForwardTypes], frameDefines, [rope("#")]) - var moduleInitRequired = false - let initname = getInitName(m.module) - var prc = "N_LIB_PRIVATE N_NIMCALL(void, $1)(void) {$N" % [initname] + var moduleInitRequired = m.hcrOn + let initname = getInitName(m) + var prc = "$1 N_NIMCALL(void, $2)(void) {$N" % + [rope(if m.hcrOn: "N_LIB_EXPORT" else: "N_LIB_PRIVATE"), initname] + # we don't want to break into such init code - could happen if a line + # directive from a function written by the user spills after itself + genCLineDir(prc, "generated_not_to_break_here", 999999, m.config) if m.typeNodes > 0: - appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", - [m.typeNodesName, rope(m.typeNodes)]) + if m.hcrOn: + appcg(m, m.s[cfsTypeInit1], "\t#TNimNode* $1;$N", [m.typeNodesName]) + appcg(m, m.s[cfsTypeInit1], "\thcrRegisterGlobal($3, \"$1_$2\", sizeof(TNimNode) * $2, NULL, (void**)&$1);$N", + [m.typeNodesName, rope(m.typeNodes), getModuleDllPath(m, m.module)]) + else: + appcg(m, m.s[cfsTypeInit1], "static #TNimNode $1[$2];$n", + [m.typeNodesName, rope(m.typeNodes)]) if m.nimTypes > 0: appcg(m, m.s[cfsTypeInit1], "static #TNimType $1[$2];$n", [m.nimTypesName, rope(m.nimTypes)]) + if m.hcrOn: + addf(prc, "\tint* nim_hcr_dummy_ = 0;$n" & + "\tNIM_BOOL nim_hcr_do_init_ = " & + "hcrRegisterGlobal($1, \"module_initialized_\", 1, NULL, (void**)&nim_hcr_dummy_);$n", + [getModuleDllPath(m, m.module)]) + + template writeSection(thing: untyped, section: TCProcSection, addHcrGuards = false) = + if m.thing.s(section).len > 0: + moduleInitRequired = true + if addHcrGuards: add(prc, "\tif (nim_hcr_do_init_) {\n\n") + add(prc, genSectionStart(section, m.config)) + add(prc, m.thing.s(section)) + add(prc, genSectionEnd(section, m.config)) + if addHcrGuards: add(prc, "\n\t} // nim_hcr_do_init_\n") + if m.preInitProc.s(cpsInit).len > 0 or m.preInitProc.s(cpsStmts).len > 0: # Give this small function its own scope addf(prc, "{$N", []) # Keep a bogus frame in case the code needs one add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - if m.preInitProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.preInitProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) - - if m.preInitProc.s(cpsInit).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.preInitProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - if m.preInitProc.s(cpsStmts).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.preInitProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(preInitProc, cpsLocals) + writeSection(preInitProc, cpsInit, m.hcrOn) + writeSection(preInitProc, cpsStmts) addf(prc, "}$N", []) # add new scope for following code, because old vcc compiler need variable @@ -1367,11 +1530,7 @@ proc genInitCode(m: BModule) = moduleInitRequired = true add(prc, initGCFrame(m.initProc)) - if m.initProc.s(cpsLocals).len > 0: - moduleInitRequired = true - add(prc, genSectionStart(cpsLocals, m.config)) - add(prc, m.initProc.s(cpsLocals)) - add(prc, genSectionEnd(cpsLocals, m.config)) + writeSection(initProc, cpsLocals) if m.initProc.s(cpsInit).len > 0 or m.initProc.s(cpsStmts).len > 0: moduleInitRequired = true @@ -1385,13 +1544,8 @@ proc genInitCode(m: BModule) = else: add(prc, ~"\tTFrame FR_; FR_.len = 0;$N") - add(prc, genSectionStart(cpsInit, m.config)) - add(prc, m.initProc.s(cpsInit)) - add(prc, genSectionEnd(cpsInit, m.config)) - - add(prc, genSectionStart(cpsStmts, m.config)) - add(prc, m.initProc.s(cpsStmts)) - add(prc, genSectionEnd(cpsStmts, m.config)) + writeSection(initProc, cpsInit, m.hcrOn) + writeSection(initProc, cpsStmts) if optStackTrace in m.initProc.options and preventStackTrace notin m.flags: add(prc, deinitFrame(m.initProc)) @@ -1407,6 +1561,19 @@ proc genInitCode(m: BModule) = # that would lead to a *nesting* of merge sections which the merger does # not support. So we add it to another special section: ``cfsInitProc`` + if m.hcrOn: + var procsToLoad = @["hcrRegisterProc", "hcrGetProc", "hcrRegisterGlobal", "hcrGetGlobal"] + + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, $1)(void* handle, N_NIMCALL_PTR(void*, getProcAddr)(void*, char*)) {$N", [getHcrInitName(m)]) + if sfMainModule in m.module.flags: + # additional procs to load + procsToLoad.add("hcrInit") + procsToLoad.add("hcrAddModule") + # load procs + for curr in procsToLoad: + add(m.s[cfsInitProc], hcrGetProcLoadCode(m, curr, "", "handle", "getProcAddr")) + addf(m.s[cfsInitProc], "}$N$N", []) + for i, el in pairs(m.extensionLoaders): if el != nil: let ex = "NIM_EXTERNC N_NIMCALL(void, nimLoadProcs$1)(void) {$2}$N$N" % @@ -1418,6 +1585,12 @@ proc genInitCode(m: BModule) = add(m.s[cfsInitProc], prc) genDatInitCode(m) + + if m.hcrOn: + addf(m.s[cfsInitProc], "N_LIB_EXPORT N_NIMCALL(void, HcrCreateTypeInfos)(void) {$N", []) + add(m.s[cfsInitProc], m.hcrCreateTypeInfosProc) + addf(m.s[cfsInitProc], "}$N$N", []) + registerModuleToMain(m.g, m) proc genModule(m: BModule, cfile: Cfile): Rope = @@ -1445,7 +1618,7 @@ proc genModule(m: BModule, cfile: Cfile): Rope = if m.s[cfsInitProc].len > 0: moduleIsEmpty = false add(result, m.s[cfsInitProc]) - if m.s[cfsDatInitProc].len > 0: + if m.s[cfsDatInitProc].len > 0 or m.hcrOn: moduleIsEmpty = false add(result, m.s[cfsDatInitProc]) @@ -1565,6 +1738,25 @@ when false: readMergeInfo(getCFile(m), m) result = m +proc addHcrInitGuards(p: BProc, n: PNode, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, inInitGuard) + else: + let stmtShouldExecute = n.kind in {nkVarSection, nkLetSection} or + nfExecuteOnReload in n.flags + if inInitGuard: + if stmtShouldExecute: + endBlock(p) + inInitGuard = false + else: + if not stmtShouldExecute: + line(p, cpsStmts, "if (nim_hcr_do_init_)\n") + startBlock(p) + inInitGuard = true + + genStmts(p, n) + proc myProcess(b: PPassContext, n: PNode): PNode = result = n if b == nil: return @@ -1573,8 +1765,11 @@ proc myProcess(b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) #softRnl = if optLineDir in m.config.options: noRnl else: rnl # XXX replicate this logic! - let tranformed_n = transformStmt(m.g.graph, m.module, n) - genStmts(m.initProc, tranformed_n) + let transformed_n = transformStmt(m.g.graph, m.module, n) + if m.hcrOn: + addHcrInitGuards(m.initProc, transformed_n, m.inHcrInitGuard) + else: + genStmts(m.initProc, transformed_n) proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = result = true @@ -1674,7 +1869,24 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = m.initProc.options = initProcOptions(m) genStmts(m.initProc, n) + if m.hcrOn: + # make sure this is pulled in (meaning hcrGetGlobal() is called for it during init) + discard cgsym(m, "programResult") + if m.inHcrInitGuard: + endBlock(m.initProc) + if sfMainModule in m.module.flags: + if m.hcrOn: + # pull ("define" since they are inline when HCR is on) these functions in the main file + # so it can load the HCR runtime and later pass the library handle to the HCR runtime which + # will in turn pass it to the other modules it initializes so they can initialize the + # register/get procs so they don't have to have the definitions of these functions as well + discard cgsym(m, "nimLoadLibrary") + discard cgsym(m, "nimLoadLibraryError") + discard cgsym(m, "nimGetProcAddr") + discard cgsym(m, "procAddrError") + discard cgsym(m, "rawWrite") + # raise dependencies on behalf of genMainProc if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone: discard cgsym(m, "initStackBottomWith") @@ -1709,14 +1921,11 @@ proc genForwardedProcs(g: BModuleList) = proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = let g = BModuleList(backend) + g.config = config + # we need to process the transitive closure because recursive module # deps are allowed (and the system module is processed in the wrong # order anyway) - g.config = config - let (outDir, _, _) = splitFile(config.outfile) - if not outDir.isEmpty: - createDir(outDir) - genForwardedProcs(g) for m in cgenModules(g): diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 50b484e31..7b1e7c123 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -98,6 +98,7 @@ type TTypeSeq* = seq[PType] TypeCache* = Table[SigHash, Rope] + TypeCacheWithOwner* = Table[SigHash, tuple[str: Rope, owner: PSym]] Codegenflag* = enum preventStackTrace, # true if stack traces need to be prevented @@ -118,7 +119,7 @@ type generatedHeader*: BModule breakPointId*: int breakpoints*: Rope # later the breakpoints are inserted into the main proc - typeInfoMarker*: TypeCache + typeInfoMarker*: TypeCacheWithOwner config*: ConfigRef graph*: ModuleGraph strVersion*, seqVersion*: int # version of the string/seq implementation to use @@ -150,6 +151,8 @@ type typeInfoMarker*: TypeCache # needed for generating type information initProc*: BProc # code for init procedure preInitProc*: BProc # code executed before the init proc + hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on + inHcrInitGuard*: bool # We are currently withing a HCR reloading guard. typeStack*: TTypeSeq # used for type generation dataCache*: TNodeTable typeNodes*, nimTypes*: int # used for type info generation @@ -189,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc = result.sigConflicts = initCountTable[string]() proc newModuleList*(g: ModuleGraph): BModuleList = - BModuleList(typeInfoMarker: initTable[SigHash, Rope](), config: g.config, - graph: g, nimtvDeclared: initIntSet()) + BModuleList(typeInfoMarker: initTable[SigHash, tuple[str: Rope, owner: PSym]](), + config: g.config, graph: g, nimtvDeclared: initIntSet()) iterator cgenModules*(g: BModuleList): BModule = for m in g.modules_closed: diff --git a/compiler/commands.nim b/compiler/commands.nim index 56fe8f205..1f46abca4 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -387,7 +387,12 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; conf.nimcacheDir = processPath(conf, arg, info, true) of "out", "o": expectArg(conf, switch, arg, pass, info) - conf.outFile = AbsoluteFile arg + let f = splitFile(arg.expandTilde) + conf.outFile = RelativeFile f.name + conf.outDir = toAbsoluteDir f.dir + of "outdir": + expectArg(conf, switch, arg, pass, info) + conf.outDir = toAbsoluteDir arg.expandTilde of "docseesrcurl": expectArg(conf, switch, arg, pass, info) conf.docSeeSrcUrl = arg @@ -487,9 +492,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; if optMemTracker in conf.options: defineSymbol(conf.symbols, "memtracker") else: undefSymbol(conf.symbols, "memtracker") of "hotcodereloading": - processOnOffSwitch(conf, {optHotCodeReloading}, arg, pass, info) - if optHotCodeReloading in conf.options: defineSymbol(conf.symbols, "hotcodereloading") - else: undefSymbol(conf.symbols, "hotcodereloading") + processOnOffSwitchG(conf, {optHotCodeReloading}, arg, pass, info) + if conf.hcrOn: + defineSymbol(conf.symbols, "hotcodereloading") + defineSymbol(conf.symbols, "useNimRtl") + else: + undefSymbol(conf.symbols, "hotcodereloading") + undefSymbol(conf.symbols, "useNimRtl") of "oldnewlines": case arg.normalize of "","on": @@ -787,7 +796,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser; if pass == passCmd1: config.commandArgs.add p.key if argsCount == 1: # support UNIX style filenames everywhere for portable build scripts: - config.projectName = unixToNativePath(p.key) + if config.projectName.len == 0: + config.projectName = unixToNativePath(p.key) config.arguments = cmdLineRest(p) result = true inc argsCount diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 5e7ce3a08..57dd55132 100644 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -82,7 +82,9 @@ proc initDefines*(symbols: StringTableRef) = defineSymbol("nimHasTypeof") defineSymbol("nimErrorProcCanHaveBody") defineSymbol("nimHasInstantiationOfInMacro") + defineSymbol("nimHasHotCodeReloading") defineSymbol("nimHasNilSeqs") + defineSymbol("nimHasSignatureHashInMacro") for f in low(Feature)..high(Feature): defineSymbol("nimHas" & $f) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5af4c464e..28d53533f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -100,14 +100,12 @@ proc parseRst(text, filename: string, proc getOutFile2(conf: ConfigRef; filename: RelativeFile, ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile = if optWholeProject in conf.globalOptions: - # This is correct, for 'nim doc --project' we interpret the '--out' option as an - # absolute directory, not as a filename! - let d = if conf.outFile.isEmpty: conf.projectPath / dir else: AbsoluteDir(conf.outFile) + let d = if conf.outDir.isEmpty: conf.projectPath / dir else: conf.outDir createDir(d) result = d / changeFileExt(filename, ext) elif guessTarget: - let d = if not conf.outFile.isEmpty: splitFile(conf.outFile).dir - else: conf.projectPath + let d = if not conf.outDir.isEmpty: conf.outDir + else: conf.projectPath createDir(d) result = d / changeFileExt(filename, ext) else: @@ -952,9 +950,8 @@ proc genOutFile(d: PDoc): Rope = proc generateIndex*(d: PDoc) = if optGenIndex in d.conf.globalOptions: - let dir = if d.conf.outFile.isEmpty: d.conf.projectPath / RelativeDir"htmldocs" - elif optWholeProject in d.conf.globalOptions: AbsoluteDir(d.conf.outFile) - else: AbsoluteDir(d.conf.outFile.string.splitFile.dir) + let dir = if not d.conf.outDir.isEmpty: d.conf.outDir + else: d.conf.projectPath / RelativeDir"htmldocs" createDir(dir) let dest = dir / changeFileExt(relativeTo(AbsoluteFile d.filename, d.conf.projectPath), IndexExt) @@ -995,6 +992,7 @@ proc writeOutputJson*(d: PDoc, useWarning = false) = "\" for writing") proc commandDoc*(cache: IdentCache, conf: ConfigRef) = + conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) var ast = parseFile(conf.projectMainIdx, cache, conf) if ast == nil: return var d = newDocumentor(conf.projectFull, cache, conf) diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim index f2ac6c1a9..2232601f4 100644 --- a/compiler/docgen2.nim +++ b/compiler/docgen2.nim @@ -29,7 +29,6 @@ template shouldProcess(g): bool = template closeImpl(body: untyped) {.dirty.} = var g = PGen(p) let useWarning = sfMainModule notin g.module.flags - #echo g.module.name.s, " ", g.module.owner.id, " ", gMainPackageId if shouldProcess(g): body try: diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 2c5af6433..18a81ca96 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -14,7 +14,7 @@ import ropes, os, strutils, osproc, platform, condsyms, options, msgs, - lineinfos, std / sha1, streams, pathutils + lineinfos, std / sha1, streams, pathutils, sequtils, times type TInfoCCProp* = enum # properties of the C compiler: @@ -468,8 +468,8 @@ proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % cmd) -proc generateScript(conf: ConfigRef; projectFile: AbsoluteFile, script: Rope) = - let (_, name, _) = splitFile(projectFile) +proc generateScript(conf: ConfigRef; script: Rope) = + let (_, name, _) = splitFile(conf.outFile.string) let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name, platform.OS[conf.target.targetOS].scriptExt)) if writeRope(script, filename): @@ -551,14 +551,14 @@ proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler else: getCompilerExe(conf, compiler, AbsoluteFile"") -proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile): string = +proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false): string = var c = conf.cCompiler var options = cFileSpecificOptions(conf, cfile.cname) var exe = getConfigVar(conf, c, ".exe") if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) if needsExeExt(conf): exe = addFileExt(exe, "exe") - if optGenDynLib in conf.globalOptions and + if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and ospNeedsPIC in platform.OS[conf.target.targetOS].props: add(options, ' ' & CC[c].pic) @@ -648,12 +648,14 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) -proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, +proc compileCFiles(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var TStringSeq, prettyCmds: var TStringSeq) = + var currIdx = 0 for it in list: # call the C compiler for the .c file: if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(conf, it) + var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1) + inc currIdx if optCompileOnly notin conf.globalOptions: add(cmds, compileCmd) let (_, name, _) = splitFile(it.cname) @@ -662,7 +664,8 @@ proc compileCFile(conf: ConfigRef; list: CFileList, script: var Rope, cmds: var add(script, compileCmd) add(script, "\n") -proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): string = +proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, + objfiles: string, isDllBuild: bool): string = if optGenStaticLib in conf.globalOptions: var libname: string if not conf.outFile.isEmpty: @@ -672,7 +675,7 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s else: libname = (libNameTmpl(conf) % splitFile(conf.projectName).name) result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname), - "objfiles", objfiles] + "objfiles", objfiles] else: var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe") if len(linkerExe) == 0: linkerExe = getLinkerExe(conf, conf.cCompiler) @@ -684,28 +687,16 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s CC[conf.cCompiler].buildGui else: "" - var exefile, builddll: string - if optGenDynLib in conf.globalOptions: - exefile = platform.OS[conf.target.targetOS].dllFrmt % splitFile(projectfile).name - builddll = CC[conf.cCompiler].buildDll - else: - exefile = splitFile(projectfile).name & platform.OS[conf.target.targetOS].exeExt - builddll = "" - if not conf.outFile.isEmpty: - exefile = conf.outFile.string.expandTilde - if not exefile.isAbsolute(): - exefile = getCurrentDir() / exefile - if not noAbsolutePaths(conf): - if not exefile.isAbsolute(): - exefile = string(splitFile(projectfile).dir / RelativeFile(exefile)) + let builddll = if isDllBuild: CC[conf.cCompiler].buildDll else: "" + let exefile = quoteShell(output) + when false: if optCDebug in conf.globalOptions: writeDebugInfo(exefile.changeFileExt("ndb")) - exefile = quoteShell(exefile) # Map files are required by Nintendo Switch compilation. They are a list # of all function calls in the library and where they come from. - let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(projectFile).name & ".map")) + let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(output).name & ".map")) let linkOptions = getLinkOptions(conf) & " " & getConfigVar(conf, conf.cCompiler, ".options.linker") @@ -723,6 +714,46 @@ proc getLinkCmd(conf: ConfigRef; projectfile: AbsoluteFile, objfiles: string): s "objfiles", objfiles, "exefile", exefile, "nim", quoteShell(getPrefixDir(conf)), "lib", quoteShell(conf.libpath)]) + # On windows the debug information for binaries is emitted in a separate .pdb + # file and the binaries (.dll and .exe) contain a full path to that .pdb file. + # This is a problem for hot code reloading because even when we copy the .dll + # and load the copy so the build process may overwrite the original .dll on + # the disk (windows locks the files of running binaries) the copy still points + # to the original .pdb (and a simple copy of the .pdb won't help). This is a + # problem when a debugger is attached to the program we are hot-reloading. + # This problem is nonexistent on Unix since there by default debug symbols + # are embedded in the binaries so loading a copy of a .so will be fine. There + # is the '/Z7' flag for the MSVC compiler to embed the debug info of source + # files into their respective .obj files but the linker still produces a .pdb + # when a final .dll or .exe is linked so the debug info isn't embedded. + # There is also the issue that even when a .dll is unloaded the debugger + # still keeps the .pdb for that .dll locked. This is a major problem and + # because of this we cannot just alternate between 2 names for a .pdb file + # when rebuilding a .dll - instead we need to accumulate differently named + # .pdb files in the nimcache folder - this is the easiest and most reliable + # way of being able to debug and rebuild the program at the same time. This + # is accomplished using the /PDB:<filename> flag (there also exists the + # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are + # atleast 5-10mb big and will quickly accumulate. There is a hacky solution: + # we could try to delete all .pdb files with a pattern and swallow exceptions. + # + # links about .pdb files and hot code reloading: + # https://ourmachinery.com/post/dll-hot-reloading-in-theory-and-practice/ + # https://ourmachinery.com/post/little-machines-working-together-part-2/ + # https://github.com/fungos/cr + # https://fungos.github.io/blog/2017/11/20/cr.h-a-simple-c-hot-reload-header-only-library/ + # on forcing the debugger to unlock a locked .pdb of an unloaded library: + # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/ + # and a bit about the .pdb format in case that is ever needed: + # https://github.com/crosire/blink + # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles + if conf.hcrOn and conf.cCompiler == ccVcc: + let t = now() + let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb" + result.add " /link /PDB:" & pdb + +template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string = + getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions) template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped): typed = try: @@ -799,7 +830,17 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = finally: removeFile(linkerArgs) -proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc getObjFilePath(conf: ConfigRef, f: CFile): string = + if noAbsolutePaths(conf): f.obj.extractFilename + else: f.obj.string + +proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): AbsoluteFile = + let basename = splitFile(objFile).name + let targetName = if isMain: basename & ".exe" + else: platform.OS[conf.target.targetOS].dllFrmt % basename + result = conf.nimcacheDir / RelativeFile(targetName) + +proc callCCompiler*(conf: ConfigRef) = var linkCmd: string if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: @@ -813,7 +854,7 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = when declared(echo): let cmd = prettyCmds[idx] if cmd != "": echo cmd - compileCFile(conf, conf.toCompile, script, cmds, prettyCmds) + compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) if optNoLinking notin conf.globalOptions: @@ -824,31 +865,64 @@ proc callCCompiler*(conf: ConfigRef; projectfile: AbsoluteFile) = add(objfiles, ' ') add(objfiles, quoteShell( addFileExt(objFile, CC[conf.cCompiler].objExt))) - for x in conf.toCompile: - let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string - add(objfiles, ' ') - add(objfiles, quoteShell(objFile)) - linkCmd = getLinkCmd(conf, projectfile, objfiles) - if optCompileOnly notin conf.globalOptions: - if defined(windows) and linkCmd.len > 8_000: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) + if conf.hcrOn: # lets assume that optCompileOnly isn't on + cmds = @[] + let mainFileIdx = conf.toCompile.len - 1 + for idx, x in conf.toCompile: + # don't relink each of the many binaries (one for each source file) if the nim code is + # cached because that would take too much time for small changes - the only downside to + # this is that if an external-to-link file changes the final target wouldn't be relinked + if x.flags.contains(CfileFlag.Cached): continue + # we pass each object file as if it is the project file - a .dll will be created for each such + # object file in the nimcache directory, and only in the case of the main project file will + # there be probably an executable (if the project is such) which will be copied out of the nimcache + let objFile = conf.getObjFilePath(x) + let buildDll = idx != mainFileIdx + let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll) + add(cmds, getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll)) + # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache + # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used + if conf.cCompiler == ccVcc: + for pdb in walkFiles(objFile & ".*.pdb"): + discard tryRemoveFile(pdb) + # execute link commands in parallel - output will be a bit different + # if it fails than that from execLinkCmd() but that doesn't matter + prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) + execCmdsInParallel(conf, cmds, prettyCb) + # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination + if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached): + let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) + var src = conf.hcrLinkTargetName(mainObjFile, true) + var dst = conf.prepareToWriteOutput + copyFileWithPermissions(src.string, dst.string) + else: + for x in conf.toCompile: + let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string + add(objfiles, ' ') + add(objfiles, quoteShell(objFile)) + let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput + else: AbsoluteFile(conf.projectName) + linkCmd = getLinkCmd(conf, mainOutput, objfiles) + if optCompileOnly notin conf.globalOptions: + if defined(windows) and linkCmd.len > 8_000: + # Windows's command line limit is about 8K (don't laugh...) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) else: linkCmd = "" if optGenScript in conf.globalOptions: add(script, linkCmd) add(script, "\n") - generateScript(conf, projectfile, script) + generateScript(conf, script) #from json import escapeJson import json -proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = +proc writeJsonBuildInstructions*(conf: ConfigRef) = template lit(x: untyped) = f.write x template str(x: untyped) = when compiles(escapeJson(x, buf)): @@ -895,7 +969,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = var buf = newStringOfCap(50) - let jsonFile = toGeneratedFile(conf, projectfile, "json") + let jsonFile = conf.nimcacheDir / RelativeFile(conf.projectName & ".json") var f: File if open(f, jsonFile.string, fmWrite): @@ -907,7 +981,7 @@ proc writeJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) lit "],\L\"linkcmd\": " - str getLinkCmd(conf, projectfile, objfiles) + str getLinkCmd(conf, conf.absOutFile, objfiles) lit "\L}\L" close(f) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index cad8fc990..e3ca6830c 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -83,7 +83,6 @@ type forwarded: seq[PSym] generatedSyms: IntSet typeInfoGenerated: IntSet - classes: seq[(PType, Rope)] unique: int # for temp identifier generation PProc = ref TProc @@ -133,7 +132,6 @@ proc newGlobals(): PGlobals = result.forwarded = @[] result.generatedSyms = initIntSet() result.typeInfoGenerated = initIntSet() - result.classes = @[] proc initCompRes(r: var TCompRes) = r.address = nil @@ -250,7 +248,7 @@ proc mangleName(m: BModule, s: PSym): Rope = result = rope(x) # From ES5 on reserved words can be used as object field names if s.kind != skField: - if optHotCodeReloading in m.config.options: + if m.config.hcrOn: # When hot reloading is enabled, we must ensure that the names # of functions and types will be preserved across rebuilds: add(result, idOrSig(s, m.module.name.s, m.sigConflicts)) @@ -1206,19 +1204,8 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) = gen(p, n.sons[0].sons[0], r) else: internalError(p.config, n.sons[0].info, "genAddr: " & $n.sons[0].kind) -proc thisParam(p: PProc; typ: PType): PType = - discard - proc attachProc(p: PProc; content: Rope; s: PSym) = - let otyp = thisParam(p, s.typ) - if otyp != nil: - for i, cls in p.g.classes: - if sameType(cls[0], otyp): - add(p.g.classes[i][1], content) - return - p.g.classes.add((otyp, content)) - else: - add(p.g.code, content) + add(p.g.code, content) proc attachProc(p: PProc; s: PSym) = let newp = genProc(p, s) @@ -1460,9 +1447,6 @@ proc genInfixCall(p: PProc, n: PNode, r: var TCompRes) = genArgs(p, n, r, 2) proc genCall(p: PProc, n: PNode, r: var TCompRes) = - if n.sons[0].kind == nkSym and thisParam(p, n.sons[0].typ) != nil: - genInfixCall(p, n, r) - return gen(p, n.sons[0], r) genArgs(p, n, r) if n.typ != nil: @@ -1606,13 +1590,14 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = s: Rope varCode: string varName = mangleName(p.module, v) - useReloadingGuard = sfGlobal in v.flags and optHotCodeReloading in p.config.options + useReloadingGuard = sfGlobal in v.flags and p.config.hcrOn if v.constraint.isNil: if useReloadingGuard: lineF(p, "var $1;$n", varName) lineF(p, "if ($1 === undefined) {$n", varName) varCode = $varName + inc p.extraIndent else: varCode = "var $2" else: @@ -1664,6 +1649,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) = lineF(p, varCode & " = $3;$n", [returnType, v.loc.r, s]) if useReloadingGuard: + dec p.extraIndent lineF(p, "}$n") proc genVarStmt(p: PProc, n: PNode) = @@ -2204,7 +2190,7 @@ proc genProc(oldProc: PProc, prc: PSym): Rope = else: result = ~"\L" - if optHotCodeReloading in p.config.options: + if p.config.hcrOn: # Here, we introduce thunks that create the equivalent of a jump table # for all global functions, because references to them may be stored # in JavaScript variables. The added indirection ensures that such @@ -2440,13 +2426,52 @@ proc genHeader(): Rope = "if (typeof Float64Array === 'undefined') Float64Array = Array;$n") % [rope(VersionAsString)] +proc addHcrInitGuards(p: PProc, n: PNode, + moduleLoadedVar: Rope, inInitGuard: var bool) = + if n.kind == nkStmtList: + for child in n: + addHcrInitGuards(p, child, moduleLoadedVar, inInitGuard) + else: + let stmtShouldExecute = n.kind in { + nkProcDef, nkFuncDef, nkMethodDef,nkConverterDef, + nkVarSection, nkLetSection} or nfExecuteOnReload in n.flags + + if inInitGuard: + if stmtShouldExecute: + dec p.extraIndent + line(p, "}\L") + inInitGuard = false + else: + if not stmtShouldExecute: + lineF(p, "if ($1 == undefined) {$n", [moduleLoadedVar]) + inc p.extraIndent + inInitGuard = true + + genStmt(p, n) + proc genModule(p: PProc, n: PNode) = if optStackTrace in p.options: add(p.body, frameCreate(p, makeJSString("module " & p.module.module.name.s), makeJSString(toFilename(p.config, p.module.module.info)))) let n_transformed = transformStmt(p.module.graph, p.module.module, n) - genStmt(p, n_transformed) + if p.config.hcrOn and n.kind == nkStmtList: + let moduleSym = p.module.module + var moduleLoadedVar = rope(moduleSym.name.s) & "_loaded" & + idOrSig(moduleSym, moduleSym.name.s, p.module.sigConflicts) + lineF(p, "var $1;$n", [moduleLoadedVar]) + var inGuardedBlock = false + + addHcrInitGuards(p, n_transformed, moduleLoadedVar, inGuardedBlock) + + if inGuardedBlock: + dec p.extraIndent + line(p, "}\L") + + lineF(p, "$1 = true;$n", [moduleLoadedVar]) + else: + genStmt(p, n_transformed) + if optStackTrace in p.options: add(p.body, frameDestroy(p)) @@ -2487,44 +2512,14 @@ proc getClassName(t: PType): Rope = if s.loc.r != nil: result = s.loc.r else: result = rope(s.name.s) -proc genClass(conf: ConfigRef; obj: PType; content: Rope; ext: string) = - let cls = getClassName(obj) - let t = skipTypes(obj, abstractPtrs) - let extends = if t.kind == tyObject and t.sons[0] != nil: - " extends " & getClassName(t.sons[0]) - else: nil - let result = ("<?php$n" & - "/* Generated by the Nim Compiler v$# */$n" & - "/* (c) " & copyrightYear & " Andreas Rumpf */$n$n" & - "require_once \"nimsystem.php\";$n" & - "class $#$# {$n$#$n}$n") % - [rope(VersionAsString), cls, extends, content] - - let outfile = changeFileExt(completeCFilePath(conf, AbsoluteFile($cls)), ext) - discard writeRopeIfNotEqual(result, outfile) - proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode = result = myProcess(b, n) var m = BModule(b) if passes.skipCodegen(m.config, n): return n if sfMainModule in m.module.flags: - let globals = PGlobals(graph.backend) - let ext = "js" - let f = if globals.classes.len == 0: toFilename(m.config, FileIndex m.module.position) - else: "nimsystem" let code = wholeCode(graph, m) - let outfile = - if not m.config.outFile.isEmpty: - if m.config.outFile.string.isAbsolute: m.config.outFile - else: AbsoluteFile(getCurrentDir() / m.config.outFile.string) - else: - changeFileExt(completeCFilePath(m.config, AbsoluteFile f), ext) - let (outDir, _, _) = splitFile(outfile) - if not outDir.isEmpty: - createDir(outDir) - discard writeRopeIfNotEqual(genHeader() & code, outfile) - for obj, content in items(globals.classes): - genClass(m.config, obj, content, ext) + let outFile = m.config.prepareToWriteOutput() + discard writeRopeIfNotEqual(genHeader() & code, outFile) proc myOpen(graph: ModuleGraph; s: PSym): PPassContext = result = newModule(graph, s) diff --git a/compiler/layouter.nim b/compiler/layouter.nim index 8605ade45..7b91b7bc4 100644 --- a/compiler/layouter.nim +++ b/compiler/layouter.nim @@ -56,12 +56,13 @@ proc openEmitter*(em: var Emitter, cache: IdentCache; em.lastLineNumber = 1 proc closeEmitter*(em: var Emitter) = - if fileExists(em.config.outFile) and readFile(em.config.outFile.string) == em.content: + let outFile = em.config.absOutFile + if fileExists(outFile) and readFile(outFile.string) == em.content: discard "do nothing, see #9499" return - var f = llStreamOpen(em.config.outFile, fmWrite) + var f = llStreamOpen(outFile, fmWrite) if f == nil: - rawMessage(em.config, errGenerated, "cannot open file: " & em.config.outFile.string) + rawMessage(em.config, errGenerated, "cannot open file: " & outFile.string) return f.llStreamWrite em.content llStreamClose(f) diff --git a/compiler/main.nim b/compiler/main.nim index c1477a22b..282f7d814 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -33,8 +33,9 @@ proc semanticPasses(g: ModuleGraph) = registerPass g, verbosePass registerPass g, semPass -proc writeDepsFile(g: ModuleGraph; project: AbsoluteFile) = - let f = open(changeFileExt(project, "deps").string, fmWrite) +proc writeDepsFile(g: ModuleGraph) = + let fname = g.config.nimcacheDir / RelativeFile(g.config.projectName & ".deps") + let f = open(fname.string, fmWrite) for m in g.modules: if m != nil: f.writeLine(toFullPath(g.config, m.position.FileIndex)) @@ -48,7 +49,7 @@ proc commandGenDepend(graph: ModuleGraph) = registerPass(graph, gendependPass) compileProject(graph) let project = graph.config.projectFull - writeDepsFile(graph, project) + writeDepsFile(graph) generateDot(graph, project) execExternalProgram(graph.config, "dot -Tpng -o" & changeFileExt(project, "png").string & @@ -62,6 +63,7 @@ proc commandCheck(graph: ModuleGraph) = when not defined(leanCompiler): proc commandDoc2(graph: ModuleGraph; json: bool) = + graph.config.outDir = AbsoluteDir(graph.config.outDir / graph.config.outFile) graph.config.errorMax = high(int) # do not stop after first error semanticPasses(graph) if json: registerPass(graph, docgen2JsonPass) @@ -71,6 +73,16 @@ when not defined(leanCompiler): proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config + + if conf.outDir.isEmpty: + conf.outDir = conf.projectPath + if conf.outFile.isEmpty: + let targetName = if optGenDynLib in conf.globalOptions: + platform.OS[conf.target.targetOS].dllFrmt % conf.projectName + else: + conf.projectName & platform.OS[conf.target.targetOS].exeExt + conf.outFile = RelativeFile targetName + extccomp.initVars(conf) semanticPasses(graph) registerPass(graph, cgenPass) @@ -80,11 +92,12 @@ proc commandCompileToC(graph: ModuleGraph) = return # issue #9933 cgenWriteModules(graph.backend, conf) if conf.cmd != cmdRun: - let proj = changeFileExt(conf.projectFull, "") - extccomp.callCCompiler(conf, proj) - extccomp.writeJsonBuildInstructions(conf, proj) + extccomp.callCCompiler(conf) + # for now we do not support writing out a .json file with the build instructions when HCR is on + if not conf.hcrOn: + extccomp.writeJsonBuildInstructions(conf) if optGenScript in graph.config.globalOptions: - writeDepsFile(graph, toGeneratedFile(conf, proj, "")) + writeDepsFile(graph) proc commandJsonScript(graph: ModuleGraph) = let proj = changeFileExt(graph.config.projectFull, "") @@ -93,6 +106,12 @@ proc commandJsonScript(graph: ModuleGraph) = when not defined(leanCompiler): proc commandCompileToJS(graph: ModuleGraph) = let conf = graph.config + + if conf.outDir.isEmpty: + conf.outDir = conf.projectPath + if conf.outFile.isEmpty: + conf.outFile = RelativeFile(conf.projectName & ".js") + #incl(gGlobalOptions, optSafeCode) setTarget(graph.config.target, osJS, cpuJS) #initDefines() @@ -102,7 +121,7 @@ when not defined(leanCompiler): registerPass(graph, JSgenPass) compileProject(graph) if optGenScript in graph.config.globalOptions: - writeDepsFile(graph, toGeneratedFile(conf, conf.projectFull, "")) + writeDepsFile(graph) proc interactivePasses(graph: ModuleGraph) = initDefines(graph.config.symbols) @@ -193,6 +212,12 @@ proc mainCommand*(graph: ModuleGraph) = quit "compiler wasn't built with JS code generator" else: conf.cmd = cmdCompileToJS + if conf.hcrOn: + # XXX: At the moment, system.nim cannot be compiled in JS mode + # with "-d:useNimRtl". The HCR option has been processed earlier + # and it has added this define implictly, so we must undo that here. + # A better solution might be to fix system.nim + undefSymbol(conf.symbols, "useNimRtl") commandCompileToJS(graph) of "doc0": when defined(leanCompiler): @@ -287,6 +312,7 @@ proc mainCommand*(graph: ModuleGraph) = (key: "project_path", val: %conf.projectFull.string), (key: "defined_symbols", val: definedSymbols), (key: "lib_paths", val: %libpaths), + (key: "outdir", val: %conf.outDir.string), (key: "out", val: %conf.outFile.string), (key: "nimcache", val: %getNimcacheDir(conf).string), (key: "hints", val: hints), diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 63fa597ff..8c081da08 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -33,6 +33,7 @@ type modules*: seq[PSym] ## indexed by int32 fileIdx packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. + importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies suggestMode*: bool # whether we are in nimsuggest mode or not. invalidTransitiveClosure: bool inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the @@ -118,6 +119,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph = result = ModuleGraph() initStrTable(result.packageSyms) result.deps = initIntSet() + result.importDeps = initTable[FileIndex, seq[FileIndex]]() result.modules = @[] result.importStack = @[] result.inclToMod = initTable[FileIndex, FileIndex]() diff --git a/compiler/modules.nim b/compiler/modules.nim index 442305a06..3d8ced35d 100644 --- a/compiler/modules.nim +++ b/compiler/modules.nim @@ -12,7 +12,7 @@ import ast, astalgo, magicsys, std / sha1, msgs, cgendata, sigmatch, options, idents, os, lexer, idgen, passes, syntaxes, llstream, modulegraphs, rod, - lineinfos, pathutils + lineinfos, pathutils, tables proc resetSystemArtifacts*(g: ModuleGraph) = magicsys.resetSysTypes(g) @@ -91,6 +91,9 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc assert graph.config != nil result = compileModule(graph, fileIdx, {}) graph.addDep(s, fileIdx) + # keep track of import relationships + if graph.config.hcrOn: + graph.importDeps.mgetOrPut(FileIndex(s.position), @[]).add(fileIdx) #if sfSystemModule in result.flags: # localError(result.info, errAttemptToRedefine, result.name.s) # restore the notes for outer module: diff --git a/compiler/nim.nim b/compiler/nim.nim index cbd9d6f39..5ca650789 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -21,9 +21,11 @@ when defined(i386) and defined(windows) and defined(vcc): import commands, lexer, condsyms, options, msgs, nversion, nimconf, ropes, extccomp, strutils, os, osproc, platform, main, parseopt, - nodejs, scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper, + scriptconfig, idents, modulegraphs, lineinfos, cmdlinehelper, pathutils +include nodejs + when hasTinyCBackend: import tccgen @@ -78,23 +80,10 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if conf.cmd == cmdRun: tccgen.run(conf.arguments) if optRun in conf.globalOptions: + var ex = quoteShell conf.absOutFile if conf.cmd == cmdCompileToJS: - var ex: string - if not conf.outFile.isEmpty: - ex = conf.outFile.prependCurDir.quoteShell - else: - ex = quoteShell( - completeCFilePath(conf, changeFileExt(conf.projectFull, "js").prependCurDir)) execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments) else: - var binPath: AbsoluteFile - if not conf.outFile.isEmpty: - # If the user specified an outFile path, use that directly. - binPath = conf.outFile.prependCurDir - else: - # Figure out ourselves a valid binary name. - binPath = changeFileExt(conf.projectFull, ExeExt).prependCurDir - var ex = quoteShell(binPath) execExternalProgram(conf, ex & ' ' & conf.arguments) when declared(GC_setMaxPause): diff --git a/compiler/nodejs.nim b/compiler/nodejs.nim index 7f9f28aaf..eaf99729d 100644 --- a/compiler/nodejs.nim +++ b/compiler/nodejs.nim @@ -6,3 +6,4 @@ proc findNodeJs*(): string = result = findExe("node") if result == "": result = findExe("iojs") + diff --git a/compiler/options.nim b/compiler/options.nim index c9f884986..037febc28 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -38,7 +38,6 @@ type # please make sure we have under 32 options # evaluation optPatterns, # en/disable pattern matching optMemTracker, - optHotCodeReloading, optLaxStrings, optNilSeqs, optOldAst @@ -81,6 +80,7 @@ type # please make sure we have under 32 options optMixedMode # true if some module triggered C++ codegen optListFullPaths # use full paths in toMsgFilename, toFilename optNoNimblePath + optHotCodeReloading optDynlibOverrideAll TGlobalOptions* = set[TGlobalOption] @@ -215,7 +215,8 @@ type packageCache*: StringTableRef searchPaths*: seq[AbsoluteDir] lazyPaths*: seq[AbsoluteDir] - outFile*: AbsoluteFile + outFile*: RelativeFile + outDir*: AbsoluteDir prefixDir*, libpath*, nimcacheDir*: AbsoluteDir dllOverrides, moduleOverrides*: StringTableRef projectName*: string # holds a name like 'nim' @@ -253,6 +254,8 @@ type severity: Severity) {.closure.} cppCustomNamespace*: string +proc hcrOn*(conf: ConfigRef): bool = return optHotCodeReloading in conf.globalOptions + template depConfigFields*(fn) {.dirty.} = fn(target) fn(options) @@ -314,7 +317,9 @@ proc newConfigRef*(): ConfigRef = packageCache: newPackageCache(), searchPaths: @[], lazyPaths: @[], - outFile: AbsoluteFile"", prefixDir: AbsoluteDir"", + outFile: RelativeFile"", + outDir: AbsoluteDir"", + prefixDir: AbsoluteDir"", libpath: AbsoluteDir"", nimcacheDir: AbsoluteDir"", dllOverrides: newStringTable(modeCaseInsensitive), moduleOverrides: newStringTable(modeStyleInsensitive), @@ -430,9 +435,6 @@ const const oKeepVariableNames* = true -template compilingLib*(conf: ConfigRef): bool = - gGlobalOptions * {optGenGuiApp, optGenDynLib} != {} - proc mainCommandArg*(conf: ConfigRef): string = ## This is intended for commands like check or parse ## which will work on the main project file unless @@ -452,8 +454,16 @@ proc setConfigVar*(conf: ConfigRef; key, val: string) = conf.configVars[key] = val proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile = - if not conf.outFile.isEmpty: result = conf.outFile - else: result = conf.projectPath / changeFileExt(filename, ext) + result = (if conf.outFile.isEmpty: conf.projectPath else: conf.outDir) / + changeFileExt(filename, ext) + +proc absOutFile*(conf: ConfigRef): AbsoluteFile = + conf.outDir / conf.outFile + +proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile = + ## Create the output directory and returns a full path to the output file + createDir conf.outDir + return conf.outDir / conf.outFile proc getPrefixDir*(conf: ConfigRef): AbsoluteDir = ## Gets the prefix dir, usually the parent directory where the binary resides. diff --git a/compiler/passes.nim b/compiler/passes.nim index f45ee03f7..8c302ce09 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -167,9 +167,9 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool { if graph.stopCompile(): break var n = parseTopLevelStmt(p) if n.kind == nkEmpty: break - if sfSystemModule notin module.flags and + if (sfSystemModule notin module.flags and ({sfNoForward, sfReorder} * module.flags != {} or - codeReordering in graph.config.features): + codeReordering in graph.config.features)): # read everything, no streaming possible var sl = newNodeI(nkStmtList, n.info) sl.add n diff --git a/compiler/pathutils.nim b/compiler/pathutils.nim index 7417845c0..cd6fb2a53 100644 --- a/compiler/pathutils.nim +++ b/compiler/pathutils.nim @@ -42,6 +42,10 @@ proc cmpPaths*(x, y: AbsoluteDir): int {.borrow.} proc createDir*(x: AbsoluteDir) {.borrow.} +proc toAbsoluteDir*(path: string): AbsoluteDir = + result = if path.isAbsolute: AbsoluteDir(path) + else: AbsoluteDir(getCurrentDir() / path) + proc `$`*(x: AnyPath): string = x.string when true: diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 9b3c66104..addc9e9f3 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -21,7 +21,7 @@ const const procPragmas* = {FirstCallConv..LastCallConv, wImportc, wExportc, wNodecl, wMagic, wNosideeffect, wSideeffect, wNoreturn, wDynlib, wHeader, - wCompilerProc, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, + wCompilerProc, wNonReloadable, wCore, wProcVar, wDeprecated, wVarargs, wCompileTime, wMerge, wBorrow, wExtern, wImportCompilerProc, wThread, wImportCpp, wImportObjC, wAsmNoStackFrame, wError, wDiscardable, wNoInit, wCodegenDecl, wGensym, wInject, wRaises, wTags, wLocks, wDelegator, wGcSafe, wOverride, @@ -31,7 +31,7 @@ const templatePragmas* = {wImmediate, wDeprecated, wError, wGensym, wInject, wDirty, wDelegator, wExportNims, wUsed, wPragma} macroPragmas* = {FirstCallConv..LastCallConv, wImmediate, wImportc, wExportc, - wNodecl, wMagic, wNosideeffect, wCompilerProc, wCore, wDeprecated, wExtern, + wNodecl, wMagic, wNosideeffect, wCompilerProc, wNonReloadable, wCore, wDeprecated, wExtern, wImportCpp, wImportObjC, wError, wDiscardable, wGensym, wInject, wDelegator, wExportNims, wUsed} iteratorPragmas* = {FirstCallConv..LastCallConv, wNosideeffect, wSideeffect, @@ -888,6 +888,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int, cppDefine(c.graph.config, sym.name.s) recordPragma(c, it, "cppdefine", sym.name.s) if sfFromGeneric notin sym.flags: markCompilerProc(c, sym) + of wNonReloadable: + sym.flags.incl sfNonReloadable of wProcVar: noVal(c, it) incl(sym.flags, sfProcvar) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index f66ec7062..46ae4941a 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -2535,19 +2535,22 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExpr(c, buildOverloadedSubscripts(n, getIdent(c.cache, "{}")), flags) of nkPragmaExpr: var - expr = n[0] pragma = n[1] pragmaName = considerQuotedIdent(c, pragma[0]) flags = flags + finalNodeFlags: TNodeFlags = {} case whichKeyword(pragmaName) of wExplain: flags.incl efExplain + of wExecuteOnReload: + finalNodeFlags.incl nfExecuteOnReload else: # what other pragmas are allowed for expressions? `likely`, `unlikely` invalidPragma(c, n) result = semExpr(c, n[0], flags) + result.flags.incl finalNodeFlags of nkPar, nkTupleConstr: case checkPar(c, n) of paNone: result = errorNode(c, n) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 246871928..5fb2fcd65 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -15,8 +15,6 @@ import nversion, platform, math, msgs, os, condsyms, idents, renderer, types, commands, magicsys, modulegraphs, strtabs, lineinfos -import system/indexerrors - proc newIntNodeT*(intVal: BiggestInt, n: PNode; g: ModuleGraph): PNode = case skipTypes(n.typ, abstractVarRange).kind of tyInt: diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index 95240867f..4247c6caa 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -260,9 +260,13 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]) = else: for i in 0..<t.len: c.hashType(t.sons[i], flags) c &= char(t.callConv) - if CoType notin flags: - if tfNoSideEffect in t.flags: c &= ".noSideEffect" - if tfThread in t.flags: c &= ".thread" + # purity of functions doesn't have to affect the mangling (which is in fact + # problematic for HCR - someone could have cached a pointer to another + # function which changes its purity and suddenly the cached pointer is danglign) + # IMHO anything that doesn't affect the overload resolution shouldn't be part of the mangling... + # if CoType notin flags: + # if tfNoSideEffect in t.flags: c &= ".noSideEffect" + # if tfThread in t.flags: c &= ".thread" if tfVarargs in t.flags: c &= ".varargs" of tyArray: c &= char(t.kind) @@ -344,6 +348,12 @@ proc hashOwner*(s: PSym): SigHash = md5Final c, result.Md5Digest +proc sigHash*(s: PSym): SigHash = + if s.kind in routineKinds and s.typ != nil: + result = hashProc(s) + else: + result = hashNonProc(s) + proc idOrSig*(s: PSym, currentModule: string, sigCollisions: var CountTable[SigHash]): Rope = if s.kind in routineKinds and s.typ != nil: diff --git a/compiler/transf.nim b/compiler/transf.nim index a26598094..ec390da9a 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -814,8 +814,9 @@ proc transformExceptBranch(c: PTransf, n: PNode): PTransNode = proc dontInlineConstant(orig, cnst: PNode): bool {.inline.} = # symbols that expand to a complex constant (array, etc.) should not be # inlined, unless it's the empty array: - result = orig.kind == nkSym and cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and - cnst.len != 0 + result = orig.kind == nkSym and + cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and + cnst.len != 0 proc commonOptimizations*(g: ModuleGraph; c: PSym, n: PNode): PNode = result = n @@ -1023,10 +1024,15 @@ proc transform(c: PTransf, n: PNode): PTransNode = when false: if oldDeferAnchor != nil: c.deferAnchor = oldDeferAnchor - var cnst = getConstExpr(c.module, PNode(result), c.graph) - # we inline constants if they are not complex constants: - if cnst != nil and not dontInlineConstant(n, cnst): - result = PTransNode(cnst) # do not miss an optimization + # Constants can be inlined here, but only if they cannot result in a cast + # in the back-end (e.g. var p: pointer = someProc) + let exprIsPointerCast = n.kind in {nkCast, nkConv, nkHiddenStdConv} and + n.typ.kind == tyPointer + if not exprIsPointerCast: + var cnst = getConstExpr(c.module, PNode(result), c.graph) + # we inline constants if they are not complex constants: + if cnst != nil and not dontInlineConstant(n, cnst): + result = PTransNode(cnst) # do not miss an optimization proc processTransf(c: PTransf, n: PNode, owner: PSym): PNode = # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip diff --git a/compiler/vm.nim b/compiler/vm.nim index 7493b008d..0922f7353 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -11,12 +11,12 @@ ## An instruction is 1-3 int32s in memory, it is a register based VM. import ast except getstr -import system/indexerrors import strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, parser, vmdeps, idents, trees, renderer, options, transf, parseutils, - vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl + vmmarshal, gorgeimpl, lineinfos, tables, btrees, macrocacheimpl, + sighashes from semfold import leValueConv, ordinalValToString from evaltempl import evalTemplate @@ -1492,6 +1492,13 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = regs[ra].node.strVal = a.sym.name.s else: stackTrace(c, tos, pc, errFieldXNotFound & "strVal") + of opcNSigHash: + decodeB(rkNode) + createStr regs[ra] + if regs[rb].node.kind != nkSym: + stackTrace(c, tos, pc, "node is not a symbol") + else: + regs[ra].node.strVal = $sigHash(regs[rb].node.sym) of opcSlurp: decodeB(rkNode) createStr regs[ra] diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index f3fac7c69..3b6a44deb 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -92,6 +92,7 @@ type opcNIdent, opcNGetType, opcNStrVal, + opcNSigHash, opcNSetIntVal, opcNSetFloatVal, opcNSetSymbol, opcNSetIdent, opcNSetType, opcNSetStrVal, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index e5e6d8735..ab2ac8707 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1224,6 +1224,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) = c.freeTemp(tmp) #genUnaryABC(c, n, dest, opcNGetType) of mNStrVal: genUnaryABC(c, n, dest, opcNStrVal) + of mNSigHash: genUnaryABC(c, n , dest, opcNSigHash) of mNSetIntVal: unused(c, n, dest) genBinaryStmt(c, n, opcNSetIntVal) diff --git a/compiler/wordrecg.nim b/compiler/wordrecg.nim index 6f78c9d6f..7a24a5812 100644 --- a/compiler/wordrecg.nim +++ b/compiler/wordrecg.nim @@ -53,6 +53,7 @@ type wFastcall, wClosure, wNoconv, wOn, wOff, wChecks, wRangechecks, wBoundchecks, wOverflowchecks, wNilchecks, wFloatchecks, wNanChecks, wInfChecks, wMoveChecks, + wNonReloadable, wExecuteOnReload, wAssertions, wPatterns, wWarnings, wHints, wOptimization, wRaises, wWrites, wReads, wSize, wEffects, wTags, wDeadCodeElimUnused, # deprecated, dead code elim always happens @@ -142,6 +143,7 @@ const "noconv", "on", "off", "checks", "rangechecks", "boundchecks", "overflowchecks", "nilchecks", "floatchecks", "nanchecks", "infchecks", "movechecks", + "nonreloadable", "executeonreload", "assertions", "patterns", "warnings", "hints", "optimization", "raises", "writes", "reads", "size", "effects", "tags", |