diff options
Diffstat (limited to 'compiler/extccomp.nim')
| -rw-r--r-- | compiler/extccomp.nim | 1394 |
1 files changed, 812 insertions, 582 deletions
diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index 4614eafe6..ce25da773 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -9,19 +9,21 @@ # Module providing functions for calling the different external C compilers # Uses some hard-wired facts about each C/C++ compiler, plus options read -# from a configuration file, to provide generalized procedures to compile +# from a lineinfos file, to provide generalized procedures to compile # nim files. -import - ropes, os, strutils, osproc, platform, condsyms, options, msgs, - securehash, streams +import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths -#from debuginfo import writeDebugInfo +import std/[os, osproc, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils] + +import std / strutils except addf + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + +import ../dist/checksums/src/checksums/sha1 type - TSystemCC* = enum - ccNone, ccGcc, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc, - ccTcc, ccPcc, ccUcc, ccIcl TInfoCCProp* = enum # properties of the C compiler: hasSwitchRange, # CC allows ranges in switch statements (GNU C) hasComputedGoto, # CC has computed goto (GNU C extension) @@ -31,6 +33,7 @@ type hasGnuAsm, # CC's asm uses the absurd GNU assembler syntax hasDeclspec, # CC has __declspec(X) hasAttribute, # CC has __attribute__((X)) + hasBuiltinUnreachable # CC has __builtin_unreachable TInfoCCProps* = set[TInfoCCProp] TInfoCC* = tuple[ name: string, # the short name of the compiler @@ -53,6 +56,8 @@ type # used on some platforms asmStmtFrmt: string, # format of ASM statement structStmtFmt: string, # Format for struct statement + produceAsm: string, # Format how to produce assembler listings + cppXsupport: string, # what to do to enable C++X support props: TInfoCCProps] # properties of the C compiler @@ -63,13 +68,16 @@ type template compiler(name, settings: untyped): untyped = proc name: TInfoCC {.compileTime.} = settings +const + gnuAsmListing = "-Wa,-acdl=$asmfile -g -fverbose-asm -masm=intel" + # GNU C and C++ Compiler compiler gcc: result = ( name: "gcc", objExt: "o", - optSpeed: " -O3 -ffast-math ", - optSize: " -Os -ffast-math ", + optSpeed: " -O3 -fno-ident", + optSize: " -Os -fno-ident", compilerExe: "gcc", cppCompiler: "g++", compileTmpl: "-c $options $include -o $objfile $file", @@ -83,10 +91,39 @@ compiler gcc: linkLibCmd: " -l$1", debug: "", pic: "-fPIC", + asmStmtFrmt: "__asm__($1);$n", + structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name + produceAsm: gnuAsmListing, + cppXsupport: "-std=gnu++17 -funsigned-char", + props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm, + hasAttribute, hasBuiltinUnreachable}) + +# GNU C and C++ Compiler +compiler nintendoSwitchGCC: + result = ( + name: "switch_gcc", + objExt: "o", + optSpeed: " -O3 ", + optSize: " -Os ", + compilerExe: "aarch64-none-elf-gcc", + cppCompiler: "aarch64-none-elf-g++", + compileTmpl: "-w -MMD -MP -MF $dfile -c $options $include -o $objfile $file", + buildGui: " -mwindows", + buildDll: " -shared", + buildLib: "aarch64-none-elf-gcc-ar rcs $libfile $objfiles", + linkerExe: "aarch64-none-elf-gcc", + linkTmpl: "$buildgui $builddll -Wl,-Map,$mapfile -o $exefile $objfiles $options", + includeCmd: " -I", + linkDirCmd: " -L", + linkLibCmd: " -l$1", + debug: "", + pic: "-fPIE", asmStmtFrmt: "asm($1);$n", structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name + produceAsm: gnuAsmListing, + cppXsupport: "-std=gnu++17 -funsigned-char", props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm, - hasAttribute}) + hasAttribute, hasBuiltinUnreachable}) # LLVM Frontend for GCC/G++ compiler llvmGcc: @@ -95,7 +132,11 @@ compiler llvmGcc: result.name = "llvm_gcc" result.compilerExe = "llvm-gcc" result.cppCompiler = "llvm-g++" - result.buildLib = "llvm-ar rcs $libfile $objfiles" + when defined(macosx) or defined(openbsd): + # `llvm-ar` not available + result.buildLib = "ar rcs $libfile $objfiles" + else: + result.buildLib = "llvm-ar rcs $libfile $objfiles" # Clang (LLVM) C/C++ Compiler compiler clang: @@ -110,16 +151,16 @@ compiler vcc: result = ( name: "vcc", objExt: "obj", - optSpeed: " /Ogityb2 /G7 /arch:SSE2 ", - optSize: " /O1 /G7 ", + optSpeed: " /Ogityb2 ", + optSize: " /O1 ", compilerExe: "cl", cppCompiler: "cl", - compileTmpl: "/c $options $include /Fo$objfile $file", - buildGui: " /link /SUBSYSTEM:WINDOWS ", + compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file", + buildGui: " /SUBSYSTEM:WINDOWS user32.lib ", buildDll: " /LD", - buildLib: "lib /OUT:$libfile $objfiles", + buildLib: "vccexe --command:lib$vccplatform /nologo /OUT:$libfile $objfiles", linkerExe: "cl", - linkTmpl: "$options $builddll /Fe$exefile $objfiles $buildgui", + linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options", includeCmd: " /I", linkDirCmd: " /LIBPATH:", linkLibCmd: " $1.lib", @@ -127,53 +168,57 @@ compiler vcc: pic: "", asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$3$n$1 $2", + produceAsm: "/Fa$asmfile", + cppXsupport: "", props: {hasCpp, hasAssume, hasDeclspec}) +# Nvidia CUDA NVCC Compiler +compiler nvcc: + result = gcc() + result.name = "nvcc" + result.compilerExe = "nvcc" + result.cppCompiler = "nvcc" + result.compileTmpl = "-c -x cu -Xcompiler=\"$options\" $include -o $objfile $file" + result.linkTmpl = "$buildgui $builddll -o $exefile $objfiles -Xcompiler=\"$options\"" + +# AMD HIPCC Compiler (rocm/cuda) +compiler hipcc: + result = clang() + result.name = "hipcc" + result.compilerExe = "hipcc" + result.cppCompiler = "hipcc" + +compiler clangcl: + result = vcc() + result.name = "clang_cl" + result.compilerExe = "clang-cl" + result.cppCompiler = "clang-cl" + result.linkerExe = "clang-cl" + result.linkTmpl = "-fuse-ld=lld " & result.linkTmpl + # Intel C/C++ Compiler compiler icl: - # Intel compilers try to imitate the native ones (gcc and msvc) - when defined(windows): - result = vcc() - else: - result = gcc() - + result = vcc() result.name = "icl" result.compilerExe = "icl" result.linkerExe = "icl" -# Local C Compiler -compiler lcc: - result = ( - name: "lcc", - objExt: "obj", - optSpeed: " -O -p6 ", - optSize: " -O -p6 ", - compilerExe: "lcc", - cppCompiler: "", - compileTmpl: "$options $include -Fo$objfile $file", - buildGui: " -subsystem windows", - buildDll: " -dll", - buildLib: "", # XXX: not supported yet - linkerExe: "lcclnk", - linkTmpl: "$options $buildgui $builddll -O $exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -g5 ", - pic: "", - asmStmtFrmt: "_asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - props: {}) +# Intel compilers try to imitate the native ones (gcc and msvc) +compiler icc: + result = gcc() + result.name = "icc" + result.compilerExe = "icc" + result.linkerExe = "icc" # Borland C Compiler compiler bcc: result = ( name: "bcc", objExt: "obj", - optSpeed: " -O2 -6 ", + optSpeed: " -O3 -6 ", optSize: " -O1 -6 ", - compilerExe: "bcc32", - cppCompiler: "", + compilerExe: "bcc32c", + cppCompiler: "cpp32c", compileTmpl: "-c $options $include -o$objfile $file", buildGui: " -tW", buildDll: " -tWD", @@ -187,55 +232,10 @@ compiler bcc: pic: "", asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$1 $2", - props: {hasCpp}) - -# Digital Mars C Compiler -compiler dmc: - result = ( - name: "dmc", - objExt: "obj", - optSpeed: " -ff -o -6 ", - optSize: " -ff -o -6 ", - compilerExe: "dmc", - cppCompiler: "", - compileTmpl: "-c $options $include -o$objfile $file", - buildGui: " -L/exet:nt/su:windows", - buildDll: " -WD", - buildLib: "", # XXX: not supported yet - linkerExe: "dmc", - linkTmpl: "$options $buildgui $builddll -o$exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -g ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$3$n$1 $2", - props: {hasCpp}) - -# Watcom C Compiler -compiler wcc: - result = ( - name: "wcc", - objExt: "obj", - optSpeed: " -ox -on -6 -d0 -fp6 -zW ", - optSize: "", - compilerExe: "wcl386", - cppCompiler: "", - compileTmpl: "-c $options $include -fo=$objfile $file", - buildGui: " -bw", - buildDll: " -bd", - buildLib: "", # XXX: not supported yet - linkerExe: "wcl386", - linkTmpl: "$options $buildgui $builddll -fe=$exefile $objfiles ", - includeCmd: " -i=", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -d2 ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - props: {hasCpp}) + produceAsm: "", + cppXsupport: "", + props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, + hasAttribute}) # Tiny C Compiler compiler tcc: @@ -247,7 +247,7 @@ compiler tcc: compilerExe: "tcc", cppCompiler: "", compileTmpl: "-c $options $include -o $objfile $file", - buildGui: "UNAVAILABLE!", + buildGui: "-Wl,-subsystem=gui", buildDll: " -shared", buildLib: "", # XXX: not supported yet linkerExe: "tcc", @@ -257,49 +257,26 @@ compiler tcc: linkLibCmd: "", # XXX: not supported yet debug: " -g ", pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - props: {hasSwitchRange, hasComputedGoto}) - -# Pelles C Compiler -compiler pcc: - # Pelles C - result = ( - name: "pcc", - objExt: "obj", - optSpeed: " -Ox ", - optSize: " -Os ", - compilerExe: "cc", - cppCompiler: "", - compileTmpl: "-c $options $include -Fo$objfile $file", - buildGui: " -SUBSYSTEM:WINDOWS", - buildDll: " -DLL", - buildLib: "", # XXX: not supported yet - linkerExe: "cc", - linkTmpl: "$options $buildgui $builddll -OUT:$exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -Zi ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", + asmStmtFrmt: "asm($1);$n", structStmtFmt: "$1 $2", - props: {}) + produceAsm: gnuAsmListing, + cppXsupport: "", + props: {hasSwitchRange, hasComputedGoto, hasGnuAsm}) # Your C Compiler -compiler ucc: +compiler envcc: result = ( - name: "ucc", + name: "env", objExt: "o", optSpeed: " -O3 ", optSize: " -O1 ", - compilerExe: "cc", + compilerExe: "", cppCompiler: "", - compileTmpl: "-c $options $include -o $objfile $file", + compileTmpl: "-c $ccenvflags $options $include -o $objfile $file", buildGui: "", buildDll: " -shared ", buildLib: "", # XXX: not supported yet - linkerExe: "cc", + linkerExe: "", linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options", includeCmd: " -I", linkDirCmd: "", # XXX: not supported yet @@ -308,585 +285,838 @@ compiler ucc: pic: "", asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$1 $2", - props: {}) + produceAsm: "", + cppXsupport: "", + props: {hasGnuAsm}) const CC*: array[succ(low(TSystemCC))..high(TSystemCC), TInfoCC] = [ gcc(), + nintendoSwitchGCC(), llvmGcc(), clang(), - lcc(), bcc(), - dmc(), - wcc(), vcc(), tcc(), - pcc(), - ucc(), - icl()] + envcc(), + icl(), + icc(), + clangcl(), + hipcc(), + nvcc()] hExt* = ".h" -var - cCompiler* = ccGcc # the used compiler - gMixedMode*: bool # true if some module triggered C++ codegen - cIncludes*: seq[string] = @[] # directories to search for included files - cLibs*: seq[string] = @[] # directories to search for lib files - cLinkedLibs*: seq[string] = @[] # libraries to link - -# implementation - -proc libNameTmpl(): string {.inline.} = - result = if targetOS == osWindows: "$1.lib" else: "lib$1.a" - -type - CfileFlag* {.pure.} = enum - Cached, ## no need to recompile this time - External ## file was introduced via .compile pragma - - Cfile* = object - cname*, obj*: string - flags*: set[CFileFlag] - CfileList = seq[Cfile] - -var - externalToLink: seq[string] = @[] # files to link in addition to the file - # we compiled - linkOptionsCmd: string = "" - compileOptionsCmd: seq[string] = @[] - linkOptions: string = "" - compileOptions: string = "" - ccompilerpath: string = "" - toCompile: CfileList = @[] +template writePrettyCmdsStderr(cmd) = + if cmd.len > 0: + flushDot(conf) + stderr.writeLine(cmd) proc nameToCC*(name: string): TSystemCC = ## Returns the kind of compiler referred to by `name`, or ccNone ## if the name doesn't refer to any known compiler. - for i in countup(succ(ccNone), high(TSystemCC)): + for i in succ(ccNone)..high(TSystemCC): if cmpIgnoreStyle(name, CC[i].name) == 0: return i result = ccNone -proc getConfigVar(c: TSystemCC, suffix: string): string = +proc listCCnames(): string = + result = "" + for i in succ(ccNone)..high(TSystemCC): + if i > succ(ccNone): result.add ", " + result.add CC[i].name + +proc isVSCompatible*(conf: ConfigRef): bool = + return conf.cCompiler == ccVcc or + conf.cCompiler == ccClangCl or + (conf.cCompiler == ccIcl and conf.target.hostOS in osDos..osWindows) + +proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given # for niminst support - let fullSuffix = - if gCmd == cmdCompileToCpp: - ".cpp" & suffix - elif gCmd == cmdCompileToOC: - ".objc" & suffix - elif gCmd == cmdCompileToJS: - ".js" & suffix - else: - suffix - - if (platform.hostOS != targetOS or platform.hostCPU != targetCPU) and - optCompileOnly notin gGlobalOptions: - let fullCCname = platform.CPU[targetCPU].name & '.' & - platform.OS[targetOS].name & '.' & + var fullSuffix = suffix + case conf.backend + of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix + of backendC: discard + of backendInvalid: + # during parsing of cfg files; we don't know the backend yet, no point in + # guessing wrong thing + return "" + + if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and + optCompileOnly notin conf.globalOptions: + let fullCCname = platform.CPU[conf.target.targetCPU].name & '.' & + platform.OS[conf.target.targetOS].name & '.' & CC[c].name & fullSuffix - result = getConfigVar(fullCCname) - if result.len == 0: - # not overriden for this cross compilation setting? - result = getConfigVar(CC[c].name & fullSuffix) + result = getConfigVar(conf, fullCCname) + if existsConfigVar(conf, fullCCname): + result = getConfigVar(conf, fullCCname) + else: + # not overridden for this cross compilation setting? + result = getConfigVar(conf, CC[c].name & fullSuffix) else: - result = getConfigVar(CC[c].name & fullSuffix) - -proc setCC*(ccname: string) = - cCompiler = nameToCC(ccname) - if cCompiler == ccNone: rawMessage(errUnknownCcompiler, ccname) - compileOptions = getConfigVar(cCompiler, ".options.always") - linkOptions = "" - ccompilerpath = getConfigVar(cCompiler, ".path") - for i in countup(low(CC), high(CC)): undefSymbol(CC[i].name) - defineSymbol(CC[cCompiler].name) + result = getConfigVar(conf, CC[c].name & fullSuffix) + +proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) = + conf.cCompiler = nameToCC(ccname) + if conf.cCompiler == ccNone: + localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, listCCnames()]) + conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always") + conf.linkOptions = "" + conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path") + for c in CC: undefSymbol(conf.symbols, c.name) + defineSymbol(conf.symbols, CC[conf.cCompiler].name) proc addOpt(dest: var string, src: string) = - if len(dest) == 0 or dest[len(dest)-1] != ' ': add(dest, " ") - add(dest, src) + if dest.len == 0 or dest[^1] != ' ': dest.add(" ") + dest.add(src) -proc addLinkOption*(option: string) = - addOpt(linkOptions, option) +proc addLinkOption*(conf: ConfigRef; option: string) = + addOpt(conf.linkOptions, option) -proc addCompileOption*(option: string) = - if strutils.find(compileOptions, option, 0) < 0: - addOpt(compileOptions, option) +proc addCompileOption*(conf: ConfigRef; option: string) = + if strutils.find(conf.compileOptions, option, 0) < 0: + addOpt(conf.compileOptions, option) -proc addLinkOptionCmd*(option: string) = - addOpt(linkOptionsCmd, option) +proc addLinkOptionCmd*(conf: ConfigRef; option: string) = + addOpt(conf.linkOptionsCmd, option) -proc addCompileOptionCmd*(option: string) = - compileOptionsCmd.add(option) +proc addCompileOptionCmd*(conf: ConfigRef; option: string) = + conf.compileOptionsCmd.add(option) -proc initVars*() = +proc initVars*(conf: ConfigRef) = # we need to define the symbol here, because ``CC`` may have never been set! - for i in countup(low(CC), high(CC)): undefSymbol(CC[i].name) - defineSymbol(CC[cCompiler].name) - addCompileOption(getConfigVar(cCompiler, ".options.always")) + for c in CC: undefSymbol(conf.symbols, c.name) + defineSymbol(conf.symbols, CC[conf.cCompiler].name) + addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always")) #addLinkOption(getConfigVar(cCompiler, ".options.linker")) - if len(ccompilerpath) == 0: - ccompilerpath = getConfigVar(cCompiler, ".path") + if conf.cCompilerPath.len == 0: + conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path") -proc completeCFilePath*(cfile: string, createSubDir: bool = true): string = - result = completeGeneratedFilePath(cfile, createSubDir) +proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile, + createSubDir: bool = true): AbsoluteFile = + ## Generate the absolute file path to the generated modules. + result = completeGeneratedFilePath(conf, cfile, createSubDir) -proc toObjFile*(filename: string): string = +proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile = # Object file for compilation - #if filename.endsWith(".cpp"): - # result = changeFileExt(filename, "cpp." & CC[cCompiler].objExt) - #else: - result = changeFileExt(filename, CC[cCompiler].objExt) + result = AbsoluteFile(filename.string & "." & CC[conf.cCompiler].objExt) -proc addFileToCompile*(cf: Cfile) = - toCompile.add(cf) +proc addFileToCompile*(conf: ConfigRef; cf: Cfile) = + conf.toCompile.add(cf) -proc resetCompilationLists* = - toCompile.setLen 0 +proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) = + let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string + var value = conf.cfileSpecificOptions.getOrDefault(key) + if strutils.find(value, option, 0) < 0: + addOpt(value, option) + conf.cfileSpecificOptions[key] = value + +proc resetCompilationLists*(conf: ConfigRef) = + conf.toCompile.setLen 0 ## XXX: we must associate these with their originating module # when the module is loaded/unloaded it adds/removes its items # That's because we still need to hash check the external files # Maybe we can do that in checkDep on the other hand? - externalToLink.setLen 0 + conf.externalToLink.setLen 0 -proc addExternalFileToLink*(filename: string) = - externalToLink.insert(filename, 0) +proc addExternalFileToLink*(conf: ConfigRef; filename: AbsoluteFile) = + conf.externalToLink.insert(filename.string, 0) -proc execWithEcho(cmd: string, msg = hintExecuting): int = - rawMessage(msg, cmd) +proc execWithEcho(conf: ConfigRef; cmd: string, msg = hintExecuting): int = + rawMessage(conf, msg, if msg == hintLinking and not(optListCmd in conf.globalOptions or conf.verbosity > 1): "" else: cmd) result = execCmd(cmd) -proc execExternalProgram*(cmd: string, msg = hintExecuting) = - if execWithEcho(cmd, msg) != 0: - rawMessage(errExecutionOfProgramFailed, cmd) +proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = + if execWithEcho(conf, cmd, msg) != 0: + rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % + cmd) -proc generateScript(projectFile: string, script: Rope) = - let (dir, name, ext) = splitFile(projectFile) - writeRope(script, dir / addFileExt("compile_" & name, - platform.OS[targetOS].scriptExt)) +proc generateScript(conf: ConfigRef; script: Rope) = + let (_, name, _) = splitFile(conf.outFile.string) + let filename = getNimcacheDir(conf) / RelativeFile(addFileExt("compile_" & name, + platform.OS[conf.target.targetOS].scriptExt)) + if not writeRope(script, filename): + rawMessage(conf, errGenerated, "could not write to file: " & filename.string) -proc getOptSpeed(c: TSystemCC): string = - result = getConfigVar(c, ".options.speed") +proc getOptSpeed(conf: ConfigRef; c: TSystemCC): string = + result = getConfigVar(conf, c, ".options.speed") if result == "": result = CC[c].optSpeed # use default settings from this file -proc getDebug(c: TSystemCC): string = - result = getConfigVar(c, ".options.debug") +proc getDebug(conf: ConfigRef; c: TSystemCC): string = + result = getConfigVar(conf, c, ".options.debug") if result == "": result = CC[c].debug # use default settings from this file -proc getOptSize(c: TSystemCC): string = - result = getConfigVar(c, ".options.size") +proc getOptSize(conf: ConfigRef; c: TSystemCC): string = + result = getConfigVar(conf, c, ".options.size") if result == "": result = CC[c].optSize # use default settings from this file -proc noAbsolutePaths: bool {.inline.} = +proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} = # We used to check current OS != specified OS, but this makes no sense # really: Cross compilation from Linux to Linux for example is entirely # reasonable. # `optGenMapping` is included here for niminst. - result = gGlobalOptions * {optGenScript, optGenMapping} != {} + # We use absolute paths for vcc / cl, see issue #19883. + let options = + if conf.cCompiler == ccVcc: + {optGenMapping} + else: + {optGenScript, optGenMapping} + result = conf.globalOptions * options != {} -proc cFileSpecificOptions(cfilename: string): string = - result = compileOptions - for option in compileOptionsCmd: +proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string = + result = conf.compileOptions + + for option in conf.compileOptionsCmd: if strutils.find(result, option, 0) < 0: addOpt(result, option) - var trunk = splitFile(cfilename).name - if optCDebug in gGlobalOptions: - var key = trunk & ".debug" - if existsConfigVar(key): addOpt(result, getConfigVar(key)) - else: addOpt(result, getDebug(cCompiler)) - if optOptimizeSpeed in gOptions: - var key = trunk & ".speed" - if existsConfigVar(key): addOpt(result, getConfigVar(key)) - else: addOpt(result, getOptSpeed(cCompiler)) - elif optOptimizeSize in gOptions: - var key = trunk & ".size" - if existsConfigVar(key): addOpt(result, getConfigVar(key)) - else: addOpt(result, getOptSize(cCompiler)) - var key = trunk & ".always" - if existsConfigVar(key): addOpt(result, getConfigVar(key)) - -proc getCompileOptions: string = - result = cFileSpecificOptions("__dummy__") - -proc getLinkOptions: string = - result = linkOptions & " " & linkOptionsCmd & " " - for linkedLib in items(cLinkedLibs): - result.add(CC[cCompiler].linkLibCmd % linkedLib.quoteShell) - for libDir in items(cLibs): - result.add(join([CC[cCompiler].linkDirCmd, libDir.quoteShell])) - -proc needsExeExt(): bool {.inline.} = - result = (optGenScript in gGlobalOptions and targetOS == osWindows) or - (platform.hostOS == osWindows) - -proc getCompilerExe(compiler: TSystemCC; cfile: string): string = - result = if gCmd == cmdCompileToCpp and not cfile.endsWith(".c"): - CC[compiler].cppCompiler - else: - CC[compiler].compilerExe + if optCDebug in conf.globalOptions: + let key = nimname & ".debug" + if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key)) + else: addOpt(result, getDebug(conf, conf.cCompiler)) + if optOptimizeSpeed in conf.options: + let key = nimname & ".speed" + if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key)) + else: addOpt(result, getOptSpeed(conf, conf.cCompiler)) + elif optOptimizeSize in conf.options: + let key = nimname & ".size" + if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key)) + else: addOpt(result, getOptSize(conf, conf.cCompiler)) + let key = nimname & ".always" + if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key)) + + addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile)) + +proc getCompileOptions(conf: ConfigRef): string = + result = cFileSpecificOptions(conf, "__dummy__", "__dummy__") + +proc vccplatform(conf: ConfigRef): string = + # VCC specific but preferable over the config hacks people + # had to do before, see #11306 + if conf.cCompiler == ccVcc: + let exe = getConfigVar(conf, conf.cCompiler, ".exe") + if "vccexe.exe" == extractFilename(exe): + result = case conf.target.targetCPU + of cpuI386: " --platform:x86" + of cpuArm: " --platform:arm" + of cpuAmd64: " --platform:amd64" + else: "" + else: + result = "" + else: + result = "" + +proc getLinkOptions(conf: ConfigRef): string = + result = conf.linkOptions & " " & conf.linkOptionsCmd & " " + for linkedLib in items(conf.cLinkedLibs): + result.add(CC[conf.cCompiler].linkLibCmd % linkedLib.quoteShell) + for libDir in items(conf.cLibs): + result.add(join([CC[conf.cCompiler].linkDirCmd, libDir.quoteShell])) + +proc needsExeExt(conf: ConfigRef): bool {.inline.} = + result = (optGenScript in conf.globalOptions and conf.target.targetOS == osWindows) or + (conf.target.hostOS == osWindows) + +proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool = + # List of possible file extensions taken from gcc + for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]: + if cfile.string.endsWith(ext): return true + false + +proc envFlags(conf: ConfigRef): string = + result = if conf.backend == backendCpp: + getEnv("CXXFLAGS") + else: + getEnv("CFLAGS") + +proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string = + if compiler == ccEnv: + result = if isCpp: + getEnv("CXX") + else: + getEnv("CC") + else: + result = if isCpp: + CC[compiler].cppCompiler + else: + CC[compiler].compilerExe if result.len == 0: - rawMessage(errCompilerDoesntSupportTarget, CC[compiler].name) + rawMessage(conf, errGenerated, + "Compiler '$1' doesn't support the requested target" % + CC[compiler].name) + +proc ccHasSaneOverflow*(conf: ConfigRef): bool = + if conf.cCompiler == ccGcc: + result = false # assume an old or crappy GCC + var exe = getConfigVar(conf, conf.cCompiler, ".exe") + if exe.len == 0: exe = CC[conf.cCompiler].compilerExe + # NOTE: should we need the full version, use -dumpfullversion + let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1) + if exitCode == 0: + var major: int = 0 + discard parseInt(s, major) + result = major >= 5 + else: + result = conf.cCompiler == ccCLang -proc getLinkerExe(compiler: TSystemCC): string = +proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe - elif gMixedMode and gCmd != cmdCompileToCpp: CC[compiler].cppCompiler - else: compiler.getCompilerExe("") - -proc getCompileCFileCmd*(cfile: Cfile): string = - var c = cCompiler - var options = cFileSpecificOptions(cfile.cname) - var exe = getConfigVar(c, ".exe") - if exe.len == 0: exe = c.getCompilerExe(cfile.cname) - - if needsExeExt(): exe = addFileExt(exe, "exe") - if optGenDynLib in gGlobalOptions and - ospNeedsPIC in platform.OS[targetOS].props: - add(options, ' ' & CC[c].pic) - - var includeCmd, compilePattern: string - if not noAbsolutePaths(): - # compute include paths: - includeCmd = CC[c].includeCmd & quoteShell(libpath) - - for includeDir in items(cIncludes): + else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp) + +proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, + isMainFile = false; produceOutput = false): string = + let + c = conf.cCompiler + isCpp = useCpp(conf, cfile.cname) + # We produce files like module.nim.cpp, so the absolute Nim filename is not + # cfile.name but `cfile.cname.changeFileExt("")`: + var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string) + if isCpp: + # needs to be prepended so that --passc:-std=c++17 can override default. + # we could avoid allocation by making cFileSpecificOptions inplace + options = CC[c].cppXsupport & ' ' & options + # If any C++ file was compiled, we need to use C++ driver for linking as well + incl conf.globalOptions, optMixedMode + + var exe = getConfigVar(conf, c, ".exe") + if exe.len == 0: exe = getCompilerExe(conf, c, isCpp) + + if needsExeExt(conf): exe = addFileExt(exe, "exe") + if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and + ospNeedsPIC in platform.OS[conf.target.targetOS].props: + options.add(' ' & CC[c].pic) + + if cfile.customArgs != "": + options.add ' ' + options.add cfile.customArgs + + var compilePattern: string + # compute include paths: + var includeCmd = CC[c].includeCmd & quoteShell(conf.libpath) + if not noAbsolutePaths(conf): + for includeDir in items(conf.cIncludes): includeCmd.add(join([CC[c].includeCmd, includeDir.quoteShell])) - compilePattern = joinPath(ccompilerpath, exe) + compilePattern = joinPath(conf.cCompilerPath, exe) else: - includeCmd = "" - compilePattern = c.getCompilerExe(cfile.cname) + compilePattern = exe + + includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)])) - var cf = if noAbsolutePaths(): extractFilename(cfile.cname) + let cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string) else: cfile.cname - var objfile = - if cfile.obj.len == 0: - if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(): - toObjFile(cf) + let objfile = + if cfile.obj.isEmpty: + if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf): + toObjFile(conf, cf).string else: - completeCFilePath(toObjFile(cf)) - elif noAbsolutePaths(): - extractFilename(cfile.obj) + completeCfilePath(conf, toObjFile(conf, cf)).string + elif noAbsolutePaths(conf): + extractFilename(cfile.obj.string) else: - cfile.obj + cfile.obj.string - objfile = quoteShell(objfile) - cf = quoteShell(cf) + # D files are required by nintendo switch libs for + # compilation. They are basically a list of all includes. + let dfile = objfile.changeFileExt(".d").quoteShell + + let cfsh = quoteShell(cf) result = quoteShell(compilePattern % [ - "file", cf, "objfile", objfile, "options", options, - "include", includeCmd, "nim", getPrefixDir(), - "nim", getPrefixDir(), "lib", libpath]) - add(result, ' ') - addf(result, CC[c].compileTmpl, [ - "file", cf, "objfile", objfile, + "dfile", dfile, + "file", cfsh, "objfile", quoteShell(objfile), "options", options, + "include", includeCmd, "nim", getPrefixDir(conf).string, + "lib", conf.libpath.string, + "ccenvflags", envFlags(conf)]) + + if optProduceAsm in conf.globalOptions: + if CC[conf.cCompiler].produceAsm.len > 0: + let asmfile = objfile.changeFileExt(".asm").quoteShell + addOpt(result, CC[conf.cCompiler].produceAsm % ["asmfile", asmfile]) + if produceOutput: + rawMessage(conf, hintUserRaw, "Produced assembler here: " & asmfile) + else: + if produceOutput: + rawMessage(conf, hintUserRaw, "Couldn't produce assembler listing " & + "for the selected C compiler: " & CC[conf.cCompiler].name) + + result.add(' ') + strutils.addf(result, CC[c].compileTmpl, [ + "dfile", dfile, + "file", cfsh, "objfile", quoteShell(objfile), "options", options, "include", includeCmd, - "nim", quoteShell(getPrefixDir()), - "nim", quoteShell(getPrefixDir()), - "lib", quoteShell(libpath)]) + "nim", quoteShell(getPrefixDir(conf)), + "lib", quoteShell(conf.libpath), + "vccplatform", vccplatform(conf), + "ccenvflags", envFlags(conf)]) -proc footprint(cfile: Cfile): SecureHash = +proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash = result = secureHash( - $secureHashFile(cfile.cname) & - platform.OS[targetOS].name & - platform.CPU[targetCPU].name & - extccomp.CC[extccomp.cCompiler].name & - getCompileCFileCmd(cfile)) - -proc externalFileChanged(cfile: Cfile): bool = - if gCmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM}: - return false - - var hashFile = toGeneratedFile(cfile.cname.withPackageName, "sha1") - var currentHash = footprint(cfile) - var f: File - if open(f, hashFile, fmRead): + $secureHashFile(cfile.cname.string) & + platform.OS[conf.target.targetOS].name & + platform.CPU[conf.target.targetCPU].name & + extccomp.CC[conf.cCompiler].name & + getCompileCFileCmd(conf, cfile)) + +proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = + if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good + + let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1") + let currentHash = footprint(conf, cfile) + var f: File = default(File) + if open(f, hashFile.string, fmRead): let oldHash = parseSecureHash(f.readLine()) close(f) result = oldHash != currentHash else: result = true if result: - if open(f, hashFile, fmWrite): + if open(f, hashFile.string, fmWrite): f.writeLine($currentHash) close(f) -proc addExternalFileToCompile*(c: var Cfile) = - if optForceFullMake notin gGlobalOptions and not externalFileChanged(c): +proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) = + # we want to generate the hash file unconditionally + let extFileChanged = externalFileChanged(conf, c) + if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and + not extFileChanged: c.flags.incl CfileFlag.Cached - toCompile.add(c) - -proc addExternalFileToCompile*(filename: string) = - var c = Cfile(cname: filename, - obj: toObjFile(completeCFilePath(changeFileExt(filename, ""), false)), + else: + # make sure Nim keeps recompiling the external file on reruns + # if compilation is not successful + discard tryRemoveFile(c.obj.string) + conf.toCompile.add(c) + +proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = + var c = Cfile(nimname: splitFile(filename).name, cname: filename, + obj: toObjFile(conf, completeCfilePath(conf, filename, false)), flags: {CfileFlag.External}) - addExternalFileToCompile(c) - -proc compileCFile(list: CFileList, script: var Rope, cmds: var TStringSeq, - prettyCmds: var TStringSeq) = - for it in list: - # call the C compiler for the .c file: - if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(it) - if optCompileOnly notin gGlobalOptions: - add(cmds, compileCmd) - let (_, name, _) = splitFile(it.cname) - add(prettyCmds, "CC: " & name) - if optGenScript in gGlobalOptions: - add(script, compileCmd) - add(script, tnl) - -proc getLinkCmd(projectfile, objfiles: string): string = - if optGenStaticLib in gGlobalOptions: - var libname: string - if options.outFile.len > 0: - libname = options.outFile.expandTilde - if not libname.isAbsolute(): - libname = getCurrentDir() / libname - else: - libname = (libNameTmpl() % splitFile(gProjectName).name) - result = CC[cCompiler].buildLib % ["libfile", libname, - "objfiles", objfiles] + addExternalFileToCompile(conf, c) + +proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, + objfiles: string, isDllBuild: bool, removeStaticFile: bool): string = + if optGenStaticLib in conf.globalOptions: + if removeStaticFile: + removeFile output # fixes: bug #16947 + result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output), + "objfiles", objfiles, + "vccplatform", vccplatform(conf)] else: - var linkerExe = getConfigVar(cCompiler, ".linkerexe") - if len(linkerExe) == 0: linkerExe = cCompiler.getLinkerExe + var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe") + if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler) # bug #6452: We must not use ``quoteShell`` here for ``linkerExe`` - if needsExeExt(): linkerExe = addFileExt(linkerExe, "exe") - if noAbsolutePaths(): result = linkerExe - else: result = joinPath(ccompilerpath, linkerExe) - let buildgui = if optGenGuiApp in gGlobalOptions: CC[cCompiler].buildGui - else: "" - var exefile, builddll: string - if optGenDynLib in gGlobalOptions: - exefile = platform.OS[targetOS].dllFrmt % splitFile(projectfile).name - builddll = CC[cCompiler].buildDll - else: - exefile = splitFile(projectfile).name & platform.OS[targetOS].exeExt - builddll = "" - if options.outFile.len > 0: - exefile = options.outFile.expandTilde - if not exefile.isAbsolute(): - exefile = getCurrentDir() / exefile - if not noAbsolutePaths(): - if not exefile.isAbsolute(): - exefile = joinPath(splitFile(projectfile).dir, exefile) + if needsExeExt(conf): linkerExe = addFileExt(linkerExe, "exe") + if noAbsolutePaths(conf): result = linkerExe + else: result = joinPath(conf.cCompilerPath, linkerExe) + let buildgui = if optGenGuiApp in conf.globalOptions and conf.target.targetOS == osWindows: + CC[conf.cCompiler].buildGui + else: + "" + let builddll = if isDllBuild: CC[conf.cCompiler].buildDll else: "" + let exefile = quoteShell(output) + when false: - if optCDebug in gGlobalOptions: + if optCDebug in conf.globalOptions: writeDebugInfo(exefile.changeFileExt("ndb")) - exefile = quoteShell(exefile) - let linkOptions = getLinkOptions() & " " & - getConfigVar(cCompiler, ".options.linker") - var linkTmpl = getConfigVar(cCompiler, ".linkTmpl") + + # Map files are required by Nintendo Switch compilation. They are a list + # of all function calls in the library and where they come from. + let mapfile = quoteShell(getNimcacheDir(conf) / RelativeFile(splitFile(output).name & ".map")) + + let linkOptions = getLinkOptions(conf) & " " & + getConfigVar(conf, conf.cCompiler, ".options.linker") + var linkTmpl = getConfigVar(conf, conf.cCompiler, ".linkTmpl") if linkTmpl.len == 0: - linkTmpl = CC[cCompiler].linkTmpl + linkTmpl = CC[conf.cCompiler].linkTmpl result = quoteShell(result % ["builddll", builddll, + "mapfile", mapfile, "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, - "exefile", exefile, "nim", getPrefixDir(), "lib", libpath]) + "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string]) result.add ' ' - addf(result, linkTmpl, ["builddll", builddll, + strutils.addf(result, linkTmpl, ["builddll", builddll, + "mapfile", mapfile, "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, "exefile", exefile, - "nim", quoteShell(getPrefixDir()), - "lib", quoteShell(libpath)]) - -template tryExceptOSErrorMessage(errorPrefix: string = "", body: untyped): typed = + "nim", quoteShell(getPrefixDir(conf)), + "lib", quoteShell(conf.libpath), + "vccplatform", vccplatform(conf)]) + # On windows the debug information for binaries is emitted in a separate .pdb + # file and the binaries (.dll and .exe) contain a full path to that .pdb file. + # This is a problem for hot code reloading because even when we copy the .dll + # and load the copy so the build process may overwrite the original .dll on + # the disk (windows locks the files of running binaries) the copy still points + # to the original .pdb (and a simple copy of the .pdb won't help). This is a + # problem when a debugger is attached to the program we are hot-reloading. + # This problem is nonexistent on Unix since there by default debug symbols + # are embedded in the binaries so loading a copy of a .so will be fine. There + # is the '/Z7' flag for the MSVC compiler to embed the debug info of source + # files into their respective .obj files but the linker still produces a .pdb + # when a final .dll or .exe is linked so the debug info isn't embedded. + # There is also the issue that even when a .dll is unloaded the debugger + # still keeps the .pdb for that .dll locked. This is a major problem and + # because of this we cannot just alternate between 2 names for a .pdb file + # when rebuilding a .dll - instead we need to accumulate differently named + # .pdb files in the nimcache folder - this is the easiest and most reliable + # way of being able to debug and rebuild the program at the same time. This + # is accomplished using the /PDB:<filename> flag (there also exists the + # /PDBALTPATH:<filename> flag). The only downside is that the .pdb files are + # at least 300kb big (when linking statically to the runtime - or else 5mb+) + # and will quickly accumulate. There is a hacky solution: we could try to + # delete all .pdb files with a pattern and swallow exceptions. + # + # links about .pdb files and hot code reloading: + # https://ourmachinery.com/post/dll-hot-reloading-in-theory-and-practice/ + # https://ourmachinery.com/post/little-machines-working-together-part-2/ + # https://github.com/fungos/cr + # https://fungos.github.io/blog/2017/11/20/cr.h-a-simple-c-hot-reload-header-only-library/ + # on forcing the debugger to unlock a locked .pdb of an unloaded library: + # https://blog.molecular-matters.com/2017/05/09/deleting-pdb-files-locked-by-visual-studio/ + # and a bit about the .pdb format in case that is ever needed: + # https://github.com/crosire/blink + # http://www.debuginfo.com/articles/debuginfomatch.html#pdbfiles + if conf.hcrOn and isVSCompatible(conf): + let t = now() + let pdb = output.string & "." & format(t, "MMMM-yyyy-HH-mm-") & $t.nanosecond & ".pdb" + result.add " /link /PDB:" & pdb + if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc: + result.add " /Zi /FS /Od" + +template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string, + removeStaticFile = false): string = + getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile) + +template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) = try: body except OSError: let ose = (ref OSError)(getCurrentException()) if errorPrefix.len > 0: - rawMessage(errGenerated, errorPrefix & " " & ose.msg & " " & $ose.errorCode) + rawMessage(conf, errGenerated, errorPrefix & " " & ose.msg & " " & $ose.errorCode) else: - rawMessage(errExecutionOfProgramFailed, ose.msg & " " & $ose.errorCode) + rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % + (ose.msg & " " & $ose.errorCode)) raise -proc execLinkCmd(linkCmd: string) = - tryExceptOSErrorMessage("invocation of external linker program failed."): - execExternalProgram(linkCmd, - if optListCmd in gGlobalOptions or gVerbosity > 1: hintExecuting else: hintLinking) +proc getExtraCmds(conf: ConfigRef; output: AbsoluteFile): seq[string] = + result = @[] + when defined(macosx): + if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions: + # if needed, add an option to skip or override location + result.add "dsymutil " & $(output).quoteShell + +proc execLinkCmd(conf: ConfigRef; linkCmd: string) = + tryExceptOSErrorMessage(conf, "invocation of external linker program failed."): + execExternalProgram(conf, linkCmd, hintLinking) -proc execCmdsInParallel(cmds: seq[string]; prettyCb: proc (idx: int)) = +proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) = let runCb = proc (idx: int, p: Process) = let exitCode = p.peekExitCode if exitCode != 0: - rawMessage(errGenerated, "execution of an external compiler program '" & - cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" & - p.outputStream.readAll.strip) - if gNumberOfProcessors == 0: gNumberOfProcessors = countProcessors() + rawMessage(conf, errGenerated, "execution of an external compiler program '" & + cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n") + if conf.numberOfProcessors == 0: conf.numberOfProcessors = countProcessors() var res = 0 - if gNumberOfProcessors <= 1: - for i in countup(0, high(cmds)): - tryExceptOSErrorMessage("invocation of external compiler program failed."): - res = execWithEcho(cmds[i]) - if res != 0: rawMessage(errExecutionOfProgramFailed, cmds[i]) + if conf.numberOfProcessors <= 1: + for i in 0..high(cmds): + tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."): + res = execWithEcho(conf, cmds[i]) + if res != 0: + rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % + cmds[i]) else: - tryExceptOSErrorMessage("invocation of external compiler program failed."): - if optListCmd in gGlobalOptions or gVerbosity > 1: - res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath, poParentStreams}, - gNumberOfProcessors, afterRunEvent=runCb) - elif gVerbosity == 1: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, - gNumberOfProcessors, prettyCb, afterRunEvent=runCb) - else: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, - gNumberOfProcessors, afterRunEvent=runCb) + tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."): + res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, + conf.numberOfProcessors, prettyCb, afterRunEvent=runCb) if res != 0: - if gNumberOfProcessors <= 1: - rawMessage(errExecutionOfProgramFailed, cmds.join()) + if conf.numberOfProcessors <= 1: + rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % + cmds.join()) + +proc linkViaResponseFile(conf: ConfigRef; cmd: string) = + # Extracting the linker.exe here is a bit hacky but the best solution + # given ``buildLib``'s design. + var i = 0 + var last = 0 + if cmd.len > 0 and cmd[0] == '"': + inc i + while i < cmd.len and cmd[i] != '"': inc i + last = i + inc i + else: + while i < cmd.len and cmd[i] != ' ': inc i + last = i + while i < cmd.len and cmd[i] == ' ': inc i + let linkerArgs = conf.projectName & "_" & "linkerArgs.txt" + let args = cmd.substr(i) + # GCC's response files don't support backslashes. Junk. + if conf.cCompiler == ccGcc or conf.cCompiler == ccCLang: + writeFile(linkerArgs, args.replace('\\', '/')) + else: + writeFile(linkerArgs, args) + try: + when defined(macosx): + execLinkCmd(conf, "xargs " & cmd.substr(0, last) & " < " & linkerArgs) + else: + execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs) + finally: + removeFile(linkerArgs) + +proc linkViaShellScript(conf: ConfigRef; cmd: string) = + let linkerScript = conf.projectName & "_" & "linkerScript.sh" + writeFile(linkerScript, cmd) + let shell = getEnv("SHELL") + try: + execLinkCmd(conf, shell & " " & linkerScript) + finally: + removeFile(linkerScript) + +proc getObjFilePath(conf: ConfigRef, f: Cfile): string = + if noAbsolutePaths(conf): f.obj.extractFilename + else: f.obj.string + +proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): AbsoluteFile = + let basename = splitFile(objFile).name + let targetName = if isMain: basename & ".exe" + else: platform.OS[conf.target.targetOS].dllFrmt % basename + result = conf.getNimcacheDir / RelativeFile(targetName) + +proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = + result = "" + if conf.hasHint(hintCC): + if optListCmd in conf.globalOptions or conf.verbosity > 1: + result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd) + else: + result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) + +proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = + # Prevent linkcmd from exceeding the maximum command line length. + # Windows's command line limit is about 8K (8191 characters) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000 + if linkCmd.len > MaxCmdLen: + when defined(macosx): + linkViaShellScript(conf, linkCmd) + else: + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) -proc callCCompiler*(projectfile: string) = +proc callCCompiler*(conf: ConfigRef) = var - linkCmd: string - if gGlobalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: + linkCmd: string = "" + extraCmds: seq[string] + if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: return # speed up that call if only compiling and no script shall be # generated #var c = cCompiler - var script: Rope = nil - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] - let prettyCb = proc (idx: int) = - echo prettyCmds[idx] - compileCFile(toCompile, script, cmds, prettyCmds) - if optCompileOnly notin gGlobalOptions: - execCmdsInParallel(cmds, prettyCb) - if optNoLinking notin gGlobalOptions: + var script: Rope = "" + var cmds: TStringSeq = default(TStringSeq) + var prettyCmds: TStringSeq = default(TStringSeq) + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + + for idx, it in conf.toCompile: + # call the C compiler for the .c file: + if CfileFlag.Cached in it.flags: continue + let compileCmd = getCompileCFileCmd(conf, it, idx == conf.toCompile.len - 1, produceOutput=true) + if optCompileOnly notin conf.globalOptions: + cmds.add(compileCmd) + prettyCmds.add displayProgressCC(conf, $it.cname, compileCmd) + if optGenScript in conf.globalOptions: + script.add(compileCmd) + script.add("\n") + + if optCompileOnly notin conf.globalOptions: + execCmdsInParallel(conf, cmds, prettyCb) + if optNoLinking notin conf.globalOptions: # call the linker: var objfiles = "" - for it in externalToLink: - let objFile = if noAbsolutePaths(): it.extractFilename else: it - add(objfiles, ' ') - add(objfiles, quoteShell( - addFileExt(objFile, CC[cCompiler].objExt))) - for x in toCompile: - add(objfiles, ' ') - add(objfiles, quoteShell(x.obj)) - - linkCmd = getLinkCmd(projectfile, objfiles) - if optCompileOnly notin gGlobalOptions: - execLinkCmd(linkCmd) + for it in conf.externalToLink: + let objFile = if noAbsolutePaths(conf): it.extractFilename else: it + objfiles.add(' ') + objfiles.add(quoteShell( + addFileExt(objFile, CC[conf.cCompiler].objExt))) + + if conf.hcrOn: # lets assume that optCompileOnly isn't on + cmds = @[] + let mainFileIdx = conf.toCompile.len - 1 + for idx, x in conf.toCompile: + # don't relink each of the many binaries (one for each source file) if the nim code is + # cached because that would take too much time for small changes - the only downside to + # this is that if an external-to-link file changes the final target wouldn't be relinked + if CfileFlag.Cached in x.flags: continue + # we pass each object file as if it is the project file - a .dll will be created for each such + # object file in the nimcache directory, and only in the case of the main project file will + # there be probably an executable (if the project is such) which will be copied out of the nimcache + let objFile = conf.getObjFilePath(x) + let buildDll = idx != mainFileIdx + let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll) + cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true)) + # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache + # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used + if isVSCompatible(conf): + for pdb in walkFiles(objFile & ".*.pdb"): + discard tryRemoveFile(pdb) + # execute link commands in parallel - output will be a bit different + # if it fails than that from execLinkCmd() but that doesn't matter + prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) + execCmdsInParallel(conf, cmds, prettyCb) + # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination + if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags: + let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) + let src = conf.hcrLinkTargetName(mainObjFile, true) + let dst = conf.prepareToWriteOutput + copyFileWithPermissions(src.string, dst.string) + else: + for x in conf.toCompile: + let objFile = if noAbsolutePaths(conf): x.obj.extractFilename else: x.obj.string + objfiles.add(' ') + objfiles.add(quoteShell(objFile)) + let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput + else: AbsoluteFile(conf.outFile) + + linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) + extraCmds = getExtraCmds(conf, mainOutput) + if optCompileOnly notin conf.globalOptions: + preventLinkCmdMaxCmdLen(conf, linkCmd) + for cmd in extraCmds: + execExternalProgram(conf, cmd, hintExecuting) else: linkCmd = "" - if optGenScript in gGlobalOptions: - add(script, linkCmd) - add(script, tnl) - generateScript(projectfile, script) - -#from json import escapeJson -import json - -proc writeJsonBuildInstructions*(projectfile: string) = - template lit(x: untyped) = f.write x - template str(x: untyped) = - when compiles(escapeJson(x, buf)): - buf.setLen 0 - escapeJson(x, buf) - f.write buf - else: - f.write escapeJson(x) - - proc cfiles(f: File; buf: var string; list: CfileList, isExternal: bool) = - var i = 0 - for it in list: - if CfileFlag.Cached in it.flags: continue - let compileCmd = getCompileCFileCmd(it) - lit "[" - str it.cname - lit ", " - str compileCmd - inc i - if i == list.len: - lit "]\L" - else: - lit "],\L" - - proc linkfiles(f: File; buf, objfiles: var string) = - for i, it in externalToLink: - let - objFile = if noAbsolutePaths(): it.extractFilename else: it - objStr = addFileExt(objFile, CC[cCompiler].objExt) - add(objfiles, ' ') - add(objfiles, objStr) - str objStr - if toCompile.len == 0 and i == externalToLink.high: - lit "\L" - else: - lit ",\L" - for i, x in toCompile: - let objStr = quoteShell(x.obj) - add(objfiles, ' ') - add(objfiles, objStr) - str objStr - if i == toCompile.high: - lit "\L" - else: - lit ",\L" - - var buf = newStringOfCap(50) - - let file = projectfile.splitFile.name - let jsonFile = toGeneratedFile(file, "json") - - var f: File - if open(f, jsonFile, fmWrite): - lit "{\"compile\":[\L" - cfiles(f, buf, toCompile, false) - lit "],\L\"link\":[\L" - var objfiles = "" - # XXX add every file here that is to link - linkfiles(f, buf, objfiles) - - lit "],\L\"linkcmd\": " - str getLinkCmd(projectfile, objfiles) - lit "\L}\L" - close(f) - -proc runJsonBuildInstructions*(projectfile: string) = - let file = projectfile.splitFile.name - let jsonFile = toGeneratedFile(file, "json") - try: - let data = json.parseFile(jsonFile) - let toCompile = data["compile"] - doAssert toCompile.kind == JArray - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] - for c in toCompile: - doAssert c.kind == JArray - doAssert c.len >= 2 - - add(cmds, c[1].getStr) - let (_, name, _) = splitFile(c[0].getStr) - add(prettyCmds, "CC: " & name) - - let prettyCb = proc (idx: int) = - echo prettyCmds[idx] - execCmdsInParallel(cmds, prettyCb) - - let linkCmd = data["linkcmd"] - doAssert linkCmd.kind == JString - execLinkCmd(linkCmd.getStr) - except: - echo getCurrentException().getStackTrace() - quit "error evaluating JSON file: " & jsonFile - -proc genMappingFiles(list: CFileList): Rope = + if optGenScript in conf.globalOptions: + script.add(linkCmd) + script.add("\n") + generateScript(conf, script) + +template hashNimExe(): string = $secureHashFile(os.getAppFilename()) + +proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = + # `outFile` is better than `projectName`, as it allows having different json + # files for a given source file compiled with different options; it also + # works out of the box with `hashMainCompilationParams`. + result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") + +const cacheVersion = "D20240927T193831" # update when `BuildCache` spec changes +type BuildCache = object + cacheVersion: string + outputFile: string + outputLastModificationTime: string + compile: seq[(string, string)] + link: seq[string] + linkcmd: string + extraCmds: seq[string] + configFiles: seq[string] # the hash shouldn't be needed + stdinInput: bool + projectIsCmd: bool + cmdInput: string + currentDir: string + cmdline: string + depfiles: seq[(string, string)] + nimexe: string + +proc writeJsonBuildInstructions*(conf: ConfigRef; deps: StringTableRef) = + var linkFiles = collect(for it in conf.externalToLink: + var it = it + if conf.noAbsolutePaths: it = it.extractFilename + it.addFileExt(CC[conf.cCompiler].objExt)) + for it in conf.toCompile: linkFiles.add it.obj.string + var bcache = BuildCache( + cacheVersion: cacheVersion, + outputFile: conf.absOutFile.string, + compile: collect(for i, it in conf.toCompile: + if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), + link: linkFiles, + linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand), + extraCmds: getExtraCmds(conf, conf.absOutFile), + stdinInput: conf.projectIsStdin, + projectIsCmd: conf.projectIsCmd, + cmdInput: conf.cmdInput, + configFiles: conf.configFiles.mapIt(it.string), + currentDir: getCurrentDir()) + if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): + bcache.cmdline = conf.commandLine + for it in conf.m.fileInfos: + let path = it.fullPath.string + if isAbsolute(path): # TODO: else? + if path in deps: + bcache.depfiles.add (path, deps[path]) + else: # backup for configs etc. + bcache.depfiles.add (path, $secureHashFile(path)) + + bcache.nimexe = hashNimExe() + if fileExists(bcache.outputFile): + bcache.outputLastModificationTime = $getLastModificationTime(bcache.outputFile) + conf.jsonBuildFile = conf.jsonBuildInstructionsFile + conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) + +proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = + result = false + if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true + var bcache: BuildCache = default(BuildCache) + try: bcache.fromJson(jsonFile.string.parseFile) + except IOError, OSError, ValueError: + stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string + return true + if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.configFiles != conf.configFiles.mapIt(it.string) or + bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or + bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or + bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true + if bcache.stdinInput or conf.projectIsStdin: return true + # xxx optimize by returning false if stdin input was the same + for (file, hash) in bcache.depfiles: + if $secureHashFile(file) != hash: return true + if bcache.outputLastModificationTime != $getLastModificationTime(bcache.outputFile): + return true + +proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = + var bcache: BuildCache = default(BuildCache) + try: bcache.fromJson(jsonFile.string.parseFile) + except ValueError, KeyError, JsonKindError: + let e = getCurrentException() + conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % + [e.msg, e.getStackTrace(), jsonFile.string] + let output = bcache.outputFile + createDir output.parentDir + let outputCurrent = $conf.absOutFile + if output != outputCurrent or bcache.cacheVersion != cacheVersion: + globalError(conf, gCmdLineInfo, + "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % + [outputCurrent, output, jsonFile.string]) + var cmds: TStringSeq = default(TStringSeq) + var prettyCmds: TStringSeq = default(TStringSeq) + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + for (name, cmd) in bcache.compile: + cmds.add cmd + prettyCmds.add displayProgressCC(conf, name, cmd) + execCmdsInParallel(conf, cmds, prettyCb) + preventLinkCmdMaxCmdLen(conf, bcache.linkcmd) + for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) + +proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = + result = "" for it in list: - addf(result, "--file:r\"$1\"$N", [rope(it.cname)]) + result.addf("--file:r\"$1\"$N", [rope(it.cname.string)]) -proc writeMapping*(gSymbolMapping: Rope) = - if optGenMapping notin gGlobalOptions: return +proc writeMapping*(conf: ConfigRef; symbolMapping: Rope) = + if optGenMapping notin conf.globalOptions: return var code = rope("[C_Files]\n") - add(code, genMappingFiles(toCompile)) - add(code, "\n[C_Compiler]\nFlags=") - add(code, strutils.escape(getCompileOptions())) + code.add(genMappingFiles(conf, conf.toCompile)) + code.add("\n[C_Compiler]\nFlags=") + code.add(strutils.escape(getCompileOptions(conf))) - add(code, "\n[Linker]\nFlags=") - add(code, strutils.escape(getLinkOptions() & " " & - getConfigVar(cCompiler, ".options.linker"))) + code.add("\n[Linker]\nFlags=") + code.add(strutils.escape(getLinkOptions(conf) & " " & + getConfigVar(conf, conf.cCompiler, ".options.linker"))) - add(code, "\n[Environment]\nlibpath=") - add(code, strutils.escape(libpath)) + code.add("\n[Environment]\nlibpath=") + code.add(strutils.escape(conf.libpath.string)) - addf(code, "\n[Symbols]$n$1", [gSymbolMapping]) - writeRope(code, joinPath(gProjectPath, "mapping.txt")) + code.addf("\n[Symbols]$n$1", [symbolMapping]) + let filename = conf.projectPath / RelativeFile"mapping.txt" + if not writeRope(code, filename): + rawMessage(conf, errGenerated, "could not write to file: " & filename.string) |