discard """ batchable: false """ # # # Nim's Runtime Library # (c) Copyright 2018 Nim Contributors # # See the file "copying.txt", included in this # distribution, for details about the copyright. # # This is the Nim hot code reloading run-time for the native targets. # # This minimal dynamic library is not subject to reloading when the # `hotCodeReloading` build mode is enabled. It's responsible for providing # a permanent memory location for all globals and procs within a program # and orchestrating the reloading. For globals, this is easily achieved # by storing them on the heap. For procs, we produce on the fly simple # trampolines that can be dynamically overwritten to jump to a different # target. In the host program, all globals and procs are first registered # here with `hcrRegisterGlobal` and `hcrRegisterProc` and then the # returned permanent locations are used in every reference to these symbols # onwards. # # Detailed description: # # When code is compiled with the hotCodeReloading option for native targets # a couple of things happen for all modules in a project: # - the useNimRtl option is forced (including when building the HCR runtime too) # - all modules of a target get built into separate shared libraries # - the smallest granularity of reloads is modules # - for each .c (or .cpp) in the corresponding nimcache folder of the project # a shared object is built with the name of the source file + DLL extension # - only the main module produces whatever the original project type intends # (again in nimcache) and is then copied to its original destination # - linking is done in parallel - just like compilation # - function calls to functions from the same project go through function pointers: # - with a few exceptions - see the nonReloadable pragma # - the forward declarations of the original functions become function # pointers as static globals with the same names # - the original function definitions get suffixed with _actual # - the function pointers get initialized with the address of the corresponding # function in the DatInit of their module through a call to either hcrRegisterProc # or hcrGetProc. When being registered, the _actual address is passed to # hcrRegisterProc and a permanent location is returned and assigned to the pointer. # This way the implementation (_actual) can change but the address for it # will be the same - this works by just updating a jump instruction (trampoline). # For functions from other modules hcrGetProc is used (after they are registered). # - globals are initialized only once and their state is preserved # - including locals with the {.global.} pragma # - their definitions are changed into pointer definitions which are initialized # in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the # size of the type that this HCR runtime should allocate) and a bool is returned # which when true triggers the initialization code for the global (only once). # Globals from other modules: a global pointer coupled with a hcrGetGlobal call. # - globals which have already been initialized cannot have their values changed # by changing their initialization - use a handler or some other mechanism # - new globals can be introduced when reloading # - top-level code (global scope) is executed only once - at the first module load # - the runtime knows every symbol's module owner (globals and procs) # - both the RTL and HCR shared libraries need to be near the program for execution # - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS) # - the main module is responsible for initializing the HCR runtime # - the main module loads the RTL and HCR shared objects # - after that a call to hcrInit() is done in the main module which triggers # the loading of all modules the main one imports, and doing that for the # dependencies of each module recursively. Basically a DFS traversal. # - then initialization takes place with several passes over all modules: # - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc # - HcrCreateTypeInfos - creates globals which will be referenced in the next pass # - DatInit - usual dat init + register/get procs and get globals # - Init - it does the following multiplexed operations: # - register globals (if already registered - then just retrieve pointer) # - execute top level scope (only if loaded for the first time) # - when modules are loaded the originally built shared libraries get copied in # the same folder and the copies are loaded instead of the original files # - a module import tree is built in the runtime (and maintained when reloading) # - hcrPerformCodeReload # - named `performCodeReload`, requires the hotcodereloading module # - explicitly

discard """
  output: '''Subobject test called
5'''
"""

type
  TClassOfTCustomObject {.pure, inheritable.} = object
    base* : ptr TClassOfTCustomObject
    className* : string
  TClassOfTobj = object of TClassOfTCustomObject
    nil
  TCustomObject = ref object {.inheritable.}
    class* : ptr TClassOfTCustomObject
  TObj = ref object of TCustomObject
    data: int

var ClassOfTObj: TClassOfTObj

proc initClassOfTObj() =
  ClassOfTObj.base = nil
  ClassOfTObj.className = "TObj"

initClassOfTObj()

proc initialize*(self: TObj) =
  self.class = addr ClassOfTObj
  # this generates wrong C code: && instead of &

proc newInstance(T: typedesc): T =
  mixin initialize
  new(result)
  initialize(result)

var o = TObj.newInstance()

type
    TestObj* = object of RootObj
        t:int
    SubObject* = object of TestObj

method test*(t:var TestObj) =
    echo "test called"

method test*(t:var SubObject) =
    echo "Subobject test called"
    t.t= 5

var a: SubObject

a.test()
echo a.t

jumpTable.setLen(len + 1) jumpTableEntryAddr = addr jumpTable[len] modules[module].procs[name] = ProcSym(jump: jumpTableEntryAddr, gen: generation) writeJump jumpTableEntryAddr, fn return jumpTableEntryAddr proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} = trace " get proc: ", module.sanitize, " ", name return modules[$module].procs.getOrDefault($name, ProcSym()).jump proc hcrRegisterGlobal*(module: cstring, name: cstring, size: Natural, gcMarker: HcrGcMarkerProc, outPtr: ptr pointer): bool {.nimhcr.} = trace " register global: ", module.sanitize, " ", name # Please note: We must allocate local copies of the strings, because the supplied # `cstring` will reside in the data segment of a DLL that will be later unloaded. # Also using a ptr pointer instead of a var pointer (an output parameter) # because for the C++ backend var parameters use references and in this use case # it is not possible to cast an int* (for example) to a void* and then pass it # to void*& since the casting yields an rvalue and references bind only to lvalues. let name = $name let module = $module modules[module].globals.withValue(name, global): trace " update global: ", name outPtr[] = global.p global.gen = generation global.markerProc = gcMarker return false do: outPtr[] = alloc0(size) modules[module].globals[name] = GlobalVarSym(p: outPtr[], gen: generation, markerProc: gcMarker) return true proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} = trace " get global: ", module.sanitize, " ", name return modules[$module].globals[$name].p proc getListOfModules(cstringArray: ptr pointer): seq[string] = var curr = cast[ptr cstring](cstringArray) while len(curr[]) > 0: result.add($curr[]) curr = cast[ptr cstring](cast[int64](curr) + sizeof(ptr cstring)) template cleanup(collection, body) = var toDelete: seq[string] for name, data in collection.pairs: if data.gen < generation: toDelete.add(name) trace "HCR Cleaning ", astToStr(collection), " :: ", name, " ", data.gen for name {.inject.} in toDelete: body proc cleanupGlobal(module: string, name: string) = var g: GlobalVarSym if modules[module].globals.take(name, g): dealloc g.p proc cleanupSymbols(module: string) = cleanup modules[module].globals: cleanupGlobal(module, name) cleanup modules[module].procs: modules[module].procs.del(name) proc unloadDll(name: string) = if modules[name].handle != nil: unloadLib(modules[name].handle) proc loadDll(name: cstring) {.nimhcr.} = let name = $name trace "HCR LOADING: ", name.sanitize if modules.contains(name): unloadDll(name) else: modules[name] = newModuleDesc() let copiedName = name & ".copy." & dllExt copyFileWithPermissions(name, copiedName) let lib = loadLib(copiedName) assert lib != nil modules[name].handle = lib modules[name].gen = generation modules[name].lastModification = getLastModificationTime(name) # update the list of imports by the module let getImportsProc = cast[proc (): ptr pointer {.nimcall.}]( checkedSymAddr(lib, "HcrGetImportedModules")) modules[name].imports = getListOfModules(getImportsProc()) # get the hash of the module let getHashProc = cast[proc (): cstring {.nimcall.}]( checkedSymAddr(lib, "HcrGetSigHash")) modules[name].hash = $getHashProc() hashToModuleMap[modules[name].hash] = name # Remove handlers for this module if reloading - they will be re-registered. # In order for them to be re-registered we need to de-register all globals # that trigger the registering of handlers through calls to hcrAddEventHandler modules[name].handlers.setLen(0) proc initHcrData(name: cstring) {.nimhcr.} = trace "HCR Hcr init: ", name.sanitize cast[proc (h: pointer, gpa: HcrProcGetter) {.nimcall.}]( checkedSymAddr(modules[$name].handle, "HcrInit000"))(hcrDynlibHandle, getProcAddr) proc initTypeInfoGlobals(name: cstring) {.nimhcr.} = trace "HCR TypeInfo globals init: ", name.sanitize cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "HcrCreateTypeInfos"))() proc initPointerData(name: cstring) {.nimhcr.} = trace "HCR Dat init: ", name.sanitize cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "DatInit000"))() proc initGlobalScope(name: cstring) {.nimhcr.} = trace "HCR Init000: ", name.sanitize # set the currently inited module - necessary for registering the before/after HCR handlers currentModule = $name cast[HcrModuleInitializer](checkedSymAddr(modules[$name].handle, "Init000"))() var modulesToInit: seq[string] = @[] var allModulesOrderedByDFS: seq[string] = @[] proc recursiveDiscovery(dlls: seq[string]) = for curr in dlls: if modules.contains(curr): # skip updating modules that have already been updated to the latest generation if modules[curr].gen >= generation: trace "HCR SKIP: ", curr.sanitize, " gen is already: ", modules[curr].gen continue # skip updating an unmodified module but continue traversing its dependencies if modules[curr].lastModification >= getLastModificationTime(curr): trace "HCR SKIP (not modified): ", curr.sanitize, " ", modules[curr].lastModification.sanitize # update generation so module doesn't get collected modules[curr].gen = generation # recurse to imported modules - they might be changed recursiveDiscovery(modules[curr].imports) allModulesOrderedByDFS.add(curr) continue loadDll(curr.cstring) # first load all dependencies of the current module and init it after that recursiveDiscovery(modules[curr].imports) allModulesOrderedByDFS.add(curr) modulesToInit.add(curr) proc initModules() = # first init the pointers to hcr functions and also do the registering of typeinfo globals for curr in modulesToInit: initHcrData(curr.cstring) initTypeInfoGlobals(curr.cstring) # for now system always gets fully inited before any other module (including when reloading) initPointerData(system.cstring) initGlobalScope(system.cstring) # proceed with the DatInit calls - for all modules - including the main one! for curr in allModulesOrderedByDFS: if curr != system: initPointerData(curr.cstring) mainDatInit() # execute top-level code (in global scope) for curr in modulesToInit: if curr != system: initGlobalScope(curr.cstring) # cleanup old symbols which are gone now for curr in modulesToInit: cleanupSymbols(curr) proc hcrInit*(moduleList: ptr pointer, main, sys: cstring, datInit: HcrModuleInitializer, handle: pointer, gpa: HcrProcGetter) {.nimhcr.} = trace "HCR INITING: ", main.sanitize, " gen: ", generation # initialize globals root = $main system = $sys mainDatInit = datInit hcrDynlibHandle = handle getProcAddr = gpa # the root is already added and we need it because symbols from it will also be registered in the HCR system modules[root].imports = getListOfModules(moduleList) modules[root].gen = high(int) # something huge so it doesn't get collected # recursively initialize all modules recursiveDiscovery(modules[root].imports) initModules() # the next module to be inited will be the root currentModule = root proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} = let module = hashToModuleMap[moduleHash] return modules[module].lastModification < getLastModificationTime(module) proc hcrReloadNeeded*(): bool {.nimhcr.} = for hash, _ in hashToModuleMap: if hcrHasModuleChanged(hash): return true return false proc hcrPerformCodeReload*() {.nimhcr.} = if not hcrReloadNeeded(): trace "HCR - no changes" return # We disable the GC during the reload, because the reloading procedures # will replace type info objects and GC marker procs. This seems to create # problems when the GC is executed while the reload is underway. # Future versions of NIMHCR won't use the GC, because all globals and the # metadata needed to access them will be placed in shared memory, so they # can be manipulated from external programs without reloading. GC_disable() defer: GC_enable() inc(generation) trace "HCR RELOADING: ", generation var traversedHandlerModules = initHashSet[string]() proc recursiveExecuteHandlers(isBefore: bool, module: string) = # do not process an already traversed module if traversedHandlerModules.containsOrIncl(module): return traversedHandlerModules.incl module # first recurse to do a DFS traversal for curr in modules[module].imports: recursiveExecuteHandlers(isBefore, curr) # and then execute the handlers - from leaf modules all the way up to the root module for curr in modules[module].handlers: if curr.isBefore == isBefore: curr.cb() # first execute the before reload handlers traversedHandlerModules.clear() recursiveExecuteHandlers(true, root) # do the reloading modulesToInit = @[] allModulesOrderedByDFS = @[] recursiveDiscovery(modules[root].imports) initModules() # execute the after reload handlers traversedHandlerModules.clear() recursiveExecuteHandlers(false, root) # collecting no longer referenced modules - based on their generation cleanup modules: cleanupSymbols(name) unloadDll(name) hashToModuleMap.del(modules[name].hash) modules.del(name) proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} = modules[currentModule].handlers.add( (isBefore: isBefore, cb: cb)) proc hcrAddModule*(module: cstring) {.nimhcr.} = if not modules.contains($module): modules[$module] = newModuleDesc() proc hcrGeneration*(): int {.nimhcr.} = generation proc hcrMarkGlobals*() {.compilerproc, exportc, dynlib, nimcall, gcsafe.} = # This is gcsafe, because it will be registered # only in the GC of the main thread. {.gcsafe.}: for _, module in modules: for _, global in module.globals: if global.markerProc != nil: global.markerProc() elif defined(hotcodereloading) or defined(testNimHcr): when not defined(js): const nimhcrLibname = when defined(windows): "nimhcr." & dllExt elif defined(macosx): "libnimhcr." & dllExt else: "libnimhcr." & dllExt {.pragma: nimhcr, compilerproc, importc, dynlib: nimhcrLibname.} proc hcrRegisterProc*(module: cstring, name: cstring, fn: pointer): pointer {.nimhcr.} proc hcrGetProc*(module: cstring, name: cstring): pointer {.nimhcr.} proc hcrRegisterGlobal*(module: cstring, name: cstring, size: Natural, gcMarker: HcrGcMarkerProc, outPtr: ptr pointer): bool {.nimhcr.} proc hcrGetGlobal*(module: cstring, name: cstring): pointer {.nimhcr.} proc hcrInit*(moduleList: ptr pointer, main, sys: cstring, datInit: HcrModuleInitializer, handle: pointer, gpa: HcrProcGetter) {.nimhcr.} proc hcrAddModule*(module: cstring) {.nimhcr.} proc hcrHasModuleChanged*(moduleHash: string): bool {.nimhcr.} proc hcrReloadNeeded*(): bool {.nimhcr.} proc hcrPerformCodeReload*() {.nimhcr.} proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) {.nimhcr.} proc hcrMarkGlobals*() {.raises: [], nimhcr, nimcall, gcsafe.} when declared(nimRegisterGlobalMarker): nimRegisterGlobalMarker(cast[GlobalMarkerProc](hcrMarkGlobals)) else: proc hcrHasModuleChanged*(moduleHash: string): bool = # TODO false proc hcrAddEventHandler*(isBefore: bool, cb: proc ()) = # TODO discard