diff options
Diffstat (limited to 'compiler/extccomp.nim')
| -rw-r--r-- | compiler/extccomp.nim | 750 |
1 files changed, 349 insertions, 401 deletions
diff --git a/compiler/extccomp.nim b/compiler/extccomp.nim index e94cf87c2..ce25da773 100644 --- a/compiler/extccomp.nim +++ b/compiler/extccomp.nim @@ -12,9 +12,16 @@ # from a lineinfos file, to provide generalized procedures to compile # nim files. -import - ropes, os, strutils, osproc, platform, condsyms, options, msgs, - lineinfos, std / sha1, streams, pathutils, sequtils, times, strtabs +import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths + +import std/[os, osproc, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils] + +import std / strutils except addf + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + +import ../dist/checksums/src/checksums/sha1 type TInfoCCProp* = enum # properties of the C compiler: @@ -26,6 +33,7 @@ type hasGnuAsm, # CC's asm uses the absurd GNU assembler syntax hasDeclspec, # CC has __declspec(X) hasAttribute, # CC has __attribute__((X)) + hasBuiltinUnreachable # CC has __builtin_unreachable TInfoCCProps* = set[TInfoCCProp] TInfoCC* = tuple[ name: string, # the short name of the compiler @@ -48,7 +56,8 @@ type # used on some platforms asmStmtFrmt: string, # format of ASM statement structStmtFmt: string, # Format for struct statement - produceAsm: string, # Format how to produce assembler listings + produceAsm: string, # Format how to produce assembler listings + cppXsupport: string, # what to do to enable C++X support props: TInfoCCProps] # properties of the C compiler @@ -82,11 +91,12 @@ compiler gcc: linkLibCmd: " -l$1", debug: "", pic: "-fPIC", - asmStmtFrmt: "asm($1);$n", + asmStmtFrmt: "__asm__($1);$n", structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name produceAsm: gnuAsmListing, + cppXsupport: "-std=gnu++17 -funsigned-char", props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm, - hasAttribute}) + hasAttribute, hasBuiltinUnreachable}) # GNU C and C++ Compiler compiler nintendoSwitchGCC: @@ -111,8 +121,9 @@ compiler nintendoSwitchGCC: asmStmtFrmt: "asm($1);$n", structStmtFmt: "$1 $3 $2 ", # struct|union [packed] $name produceAsm: gnuAsmListing, + cppXsupport: "-std=gnu++17 -funsigned-char", props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasGnuAsm, - hasAttribute}) + hasAttribute, hasBuiltinUnreachable}) # LLVM Frontend for GCC/G++ compiler llvmGcc: @@ -121,8 +132,8 @@ compiler llvmGcc: result.name = "llvm_gcc" result.compilerExe = "llvm-gcc" result.cppCompiler = "llvm-g++" - when defined(macosx): - # OS X has no 'llvm-ar' tool: + when defined(macosx) or defined(openbsd): + # `llvm-ar` not available result.buildLib = "ar rcs $libfile $objfiles" else: result.buildLib = "llvm-ar rcs $libfile $objfiles" @@ -140,16 +151,16 @@ compiler vcc: result = ( name: "vcc", objExt: "obj", - optSpeed: " /Ogityb2 /G7 ", - optSize: " /O1 /G7 ", + optSpeed: " /Ogityb2 ", + optSize: " /O1 ", compilerExe: "cl", cppCompiler: "cl", - compileTmpl: "/c$vccplatform $options $include /Fo$objfile $file", - buildGui: " /link /SUBSYSTEM:WINDOWS ", + compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file", + buildGui: " /SUBSYSTEM:WINDOWS user32.lib ", buildDll: " /LD", - buildLib: "lib /OUT:$libfile $objfiles", + buildLib: "vccexe --command:lib$vccplatform /nologo /OUT:$libfile $objfiles", linkerExe: "cl", - linkTmpl: "$options $builddll$vccplatform /Fe$exefile $objfiles $buildgui", + linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options", includeCmd: " /I", linkDirCmd: " /LIBPATH:", linkLibCmd: " $1.lib", @@ -158,14 +169,32 @@ compiler vcc: asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$3$n$1 $2", produceAsm: "/Fa$asmfile", + cppXsupport: "", props: {hasCpp, hasAssume, hasDeclspec}) +# Nvidia CUDA NVCC Compiler +compiler nvcc: + result = gcc() + result.name = "nvcc" + result.compilerExe = "nvcc" + result.cppCompiler = "nvcc" + result.compileTmpl = "-c -x cu -Xcompiler=\"$options\" $include -o $objfile $file" + result.linkTmpl = "$buildgui $builddll -o $exefile $objfiles -Xcompiler=\"$options\"" + +# AMD HIPCC Compiler (rocm/cuda) +compiler hipcc: + result = clang() + result.name = "hipcc" + result.compilerExe = "hipcc" + result.cppCompiler = "hipcc" + compiler clangcl: result = vcc() result.name = "clang_cl" result.compilerExe = "clang-cl" result.cppCompiler = "clang-cl" result.linkerExe = "clang-cl" + result.linkTmpl = "-fuse-ld=lld " & result.linkTmpl # Intel C/C++ Compiler compiler icl: @@ -181,31 +210,6 @@ compiler icc: result.compilerExe = "icc" result.linkerExe = "icc" -# Local C Compiler -compiler lcc: - result = ( - name: "lcc", - objExt: "obj", - optSpeed: " -O -p6 ", - optSize: " -O -p6 ", - compilerExe: "lcc", - cppCompiler: "", - compileTmpl: "$options $include -Fo$objfile $file", - buildGui: " -subsystem windows", - buildDll: " -dll", - buildLib: "", # XXX: not supported yet - linkerExe: "lcclnk", - linkTmpl: "$options $buildgui $builddll -O $exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -g5 ", - pic: "", - asmStmtFrmt: "_asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - produceAsm: "", - props: {}) - # Borland C Compiler compiler bcc: result = ( @@ -229,59 +233,10 @@ compiler bcc: asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$1 $2", produceAsm: "", + cppXsupport: "", props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard, hasAttribute}) -# Digital Mars C Compiler -compiler dmc: - result = ( - name: "dmc", - objExt: "obj", - optSpeed: " -ff -o -6 ", - optSize: " -ff -o -6 ", - compilerExe: "dmc", - cppCompiler: "", - compileTmpl: "-c $options $include -o$objfile $file", - buildGui: " -L/exet:nt/su:windows", - buildDll: " -WD", - buildLib: "", # XXX: not supported yet - linkerExe: "dmc", - linkTmpl: "$options $buildgui $builddll -o$exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -g ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$3$n$1 $2", - produceAsm: "", - props: {hasCpp}) - -# Watcom C Compiler -compiler wcc: - result = ( - name: "wcc", - objExt: "obj", - optSpeed: " -ox -on -6 -d0 -fp6 -zW ", - optSize: "", - compilerExe: "wcl386", - cppCompiler: "", - compileTmpl: "-c $options $include -fo=$objfile $file", - buildGui: " -bw", - buildDll: " -bd", - buildLib: "", # XXX: not supported yet - linkerExe: "wcl386", - linkTmpl: "$options $buildgui $builddll -fe=$exefile $objfiles ", - includeCmd: " -i=", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -d2 ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - produceAsm: "", - props: {hasCpp}) - # Tiny C Compiler compiler tcc: result = ( @@ -305,48 +260,23 @@ compiler tcc: asmStmtFrmt: "asm($1);$n", structStmtFmt: "$1 $2", produceAsm: gnuAsmListing, + cppXsupport: "", props: {hasSwitchRange, hasComputedGoto, hasGnuAsm}) -# Pelles C Compiler -compiler pcc: - # Pelles C - result = ( - name: "pcc", - objExt: "obj", - optSpeed: " -Ox ", - optSize: " -Os ", - compilerExe: "cc", - cppCompiler: "", - compileTmpl: "-c $options $include -Fo$objfile $file", - buildGui: " -SUBSYSTEM:WINDOWS", - buildDll: " -DLL", - buildLib: "", # XXX: not supported yet - linkerExe: "cc", - linkTmpl: "$options $buildgui $builddll -OUT:$exefile $objfiles", - includeCmd: " -I", - linkDirCmd: "", # XXX: not supported yet - linkLibCmd: "", # XXX: not supported yet - debug: " -Zi ", - pic: "", - asmStmtFrmt: "__asm{$n$1$n}$n", - structStmtFmt: "$1 $2", - produceAsm: "", - props: {}) - # Your C Compiler -compiler ucc: +compiler envcc: result = ( - name: "ucc", + name: "env", objExt: "o", optSpeed: " -O3 ", optSize: " -O1 ", - compilerExe: "cc", + compilerExe: "", cppCompiler: "", - compileTmpl: "-c $options $include -o $objfile $file", + compileTmpl: "-c $ccenvflags $options $include -o $objfile $file", buildGui: "", buildDll: " -shared ", buildLib: "", # XXX: not supported yet - linkerExe: "cc", + linkerExe: "", linkTmpl: "-o $exefile $buildgui $builddll $objfiles $options", includeCmd: " -I", linkDirCmd: "", # XXX: not supported yet @@ -356,7 +286,8 @@ compiler ucc: asmStmtFrmt: "__asm{$n$1$n}$n", structStmtFmt: "$1 $2", produceAsm: "", - props: {}) + cppXsupport: "", + props: {hasGnuAsm}) const CC*: array[succ(low(TSystemCC))..high(TSystemCC), TInfoCC] = [ @@ -364,22 +295,22 @@ const nintendoSwitchGCC(), llvmGcc(), clang(), - lcc(), bcc(), - dmc(), - wcc(), vcc(), tcc(), - pcc(), - ucc(), + envcc(), icl(), icc(), - clangcl()] + clangcl(), + hipcc(), + nvcc()] hExt* = ".h" -proc libNameTmpl(conf: ConfigRef): string {.inline.} = - result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a" +template writePrettyCmdsStderr(cmd) = + if cmd.len > 0: + flushDot(conf) + stderr.writeLine(cmd) proc nameToCC*(name: string): TSystemCC = ## Returns the kind of compiler referred to by `name`, or ccNone @@ -389,8 +320,10 @@ proc nameToCC*(name: string): TSystemCC = return i result = ccNone -proc listCCnames(): seq[string] = +proc listCCnames(): string = + result = "" for i in succ(ccNone)..high(TSystemCC): + if i > succ(ccNone): result.add ", " result.add CC[i].name proc isVSCompatible*(conf: ConfigRef): bool = @@ -401,22 +334,24 @@ proc isVSCompatible*(conf: ConfigRef): bool = proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given # for niminst support - let fullSuffix = - if conf.cmd == cmdCompileToCpp: - ".cpp" & suffix - elif conf.cmd == cmdCompileToOC: - ".objc" & suffix - elif conf.cmd == cmdCompileToJS: - ".js" & suffix - else: - suffix - - if optCompileOnly notin conf.globalOptions: + var fullSuffix = suffix + case conf.backend + of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix + of backendC: discard + of backendInvalid: + # during parsing of cfg files; we don't know the backend yet, no point in + # guessing wrong thing + return "" + + if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and + optCompileOnly notin conf.globalOptions: let fullCCname = platform.CPU[conf.target.targetCPU].name & '.' & platform.OS[conf.target.targetOS].name & '.' & CC[c].name & fullSuffix result = getConfigVar(conf, fullCCname) - if result.len == 0: + if existsConfigVar(conf, fullCCname): + result = getConfigVar(conf, fullCCname) + else: # not overridden for this cross compilation setting? result = getConfigVar(conf, CC[c].name & fullSuffix) else: @@ -425,12 +360,11 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string = proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) = conf.cCompiler = nameToCC(ccname) if conf.cCompiler == ccNone: - let ccList = listCCnames().join(", ") - localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, ccList]) + localError(conf, info, "unknown C compiler: '$1'. Available options are: $2" % [ccname, listCCnames()]) conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always") conf.linkOptions = "" conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path") - for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name) + for c in CC: undefSymbol(conf.symbols, c.name) defineSymbol(conf.symbols, CC[conf.cCompiler].name) proc addOpt(dest: var string, src: string) = @@ -452,7 +386,7 @@ proc addCompileOptionCmd*(conf: ConfigRef; option: string) = proc initVars*(conf: ConfigRef) = # we need to define the symbol here, because ``CC`` may have never been set! - for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name) + for c in CC: undefSymbol(conf.symbols, c.name) defineSymbol(conf.symbols, CC[conf.cCompiler].name) addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always")) #addLinkOption(getConfigVar(cCompiler, ".options.linker")) @@ -461,6 +395,7 @@ proc initVars*(conf: ConfigRef) = proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile, createSubDir: bool = true): AbsoluteFile = + ## Generate the absolute file path to the generated modules. result = completeGeneratedFilePath(conf, cfile, createSubDir) proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile = @@ -471,7 +406,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) = conf.toCompile.add(cf) proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) = - let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string + let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string var value = conf.cfileSpecificOptions.getOrDefault(key) if strutils.find(value, option, 0) < 0: addOpt(value, option) @@ -489,7 +424,7 @@ proc addExternalFileToLink*(conf: ConfigRef; filename: AbsoluteFile) = conf.externalToLink.insert(filename.string, 0) proc execWithEcho(conf: ConfigRef; cmd: string, msg = hintExecuting): int = - rawMessage(conf, msg, cmd) + rawMessage(conf, msg, if msg == hintLinking and not(optListCmd in conf.globalOptions or conf.verbosity > 1): "" else: cmd) result = execCmd(cmd) proc execExternalProgram*(conf: ConfigRef; cmd: string, msg = hintExecuting) = @@ -524,11 +459,16 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} = # really: Cross compilation from Linux to Linux for example is entirely # reasonable. # `optGenMapping` is included here for niminst. - result = conf.globalOptions * {optGenScript, optGenMapping} != {} + # We use absolute paths for vcc / cl, see issue #19883. + let options = + if conf.cCompiler == ccVcc: + {optGenMapping} + else: + {optGenScript, optGenMapping} + result = conf.globalOptions * options != {} proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string = result = conf.compileOptions - addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile)) for option in conf.compileOptionsCmd: if strutils.find(result, option, 0) < 0: @@ -549,13 +489,15 @@ proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string let key = nimname & ".always" if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key)) + addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile)) + proc getCompileOptions(conf: ConfigRef): string = result = cFileSpecificOptions(conf, "__dummy__", "__dummy__") proc vccplatform(conf: ConfigRef): string = # VCC specific but preferable over the config hacks people # had to do before, see #11306 - if conf.cCompiler == ccVcc: + if conf.cCompiler == ccVcc: let exe = getConfigVar(conf, conf.cCompiler, ".exe") if "vccexe.exe" == extractFilename(exe): result = case conf.target.targetCPU @@ -563,6 +505,10 @@ proc vccplatform(conf: ConfigRef): string = of cpuArm: " --platform:arm" of cpuAmd64: " --platform:amd64" else: "" + else: + result = "" + else: + result = "" proc getLinkOptions(conf: ConfigRef): string = result = conf.linkOptions & " " & conf.linkOptionsCmd & " " @@ -575,35 +521,79 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} = result = (optGenScript in conf.globalOptions and conf.target.targetOS == osWindows) or (conf.target.hostOS == osWindows) -proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string = - result = if conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c"): - CC[compiler].cppCompiler - else: - CC[compiler].compilerExe +proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool = + # List of possible file extensions taken from gcc + for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]: + if cfile.string.endsWith(ext): return true + false + +proc envFlags(conf: ConfigRef): string = + result = if conf.backend == backendCpp: + getEnv("CXXFLAGS") + else: + getEnv("CFLAGS") + +proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string = + if compiler == ccEnv: + result = if isCpp: + getEnv("CXX") + else: + getEnv("CC") + else: + result = if isCpp: + CC[compiler].cppCompiler + else: + CC[compiler].compilerExe if result.len == 0: rawMessage(conf, errGenerated, "Compiler '$1' doesn't support the requested target" % CC[compiler].name) +proc ccHasSaneOverflow*(conf: ConfigRef): bool = + if conf.cCompiler == ccGcc: + result = false # assume an old or crappy GCC + var exe = getConfigVar(conf, conf.cCompiler, ".exe") + if exe.len == 0: exe = CC[conf.cCompiler].compilerExe + # NOTE: should we need the full version, use -dumpfullversion + let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except IOError, OSError, ValueError: ("", 1) + if exitCode == 0: + var major: int = 0 + discard parseInt(s, major) + result = major >= 5 + else: + result = conf.cCompiler == ccCLang + proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string = result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe - elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler - else: getCompilerExe(conf, compiler, AbsoluteFile"") + else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp) proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, isMainFile = false; produceOutput = false): string = - let c = conf.cCompiler + let + c = conf.cCompiler + isCpp = useCpp(conf, cfile.cname) # We produce files like module.nim.cpp, so the absolute Nim filename is not # cfile.name but `cfile.cname.changeFileExt("")`: var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string) + if isCpp: + # needs to be prepended so that --passc:-std=c++17 can override default. + # we could avoid allocation by making cFileSpecificOptions inplace + options = CC[c].cppXsupport & ' ' & options + # If any C++ file was compiled, we need to use C++ driver for linking as well + incl conf.globalOptions, optMixedMode + var exe = getConfigVar(conf, c, ".exe") - if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname) + if exe.len == 0: exe = getCompilerExe(conf, c, isCpp) if needsExeExt(conf): exe = addFileExt(exe, "exe") if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and ospNeedsPIC in platform.OS[conf.target.targetOS].props: options.add(' ' & CC[c].pic) + if cfile.customArgs != "": + options.add ' ' + options.add cfile.customArgs + var compilePattern: string # compute include paths: var includeCmd = CC[c].includeCmd & quoteShell(conf.libpath) @@ -613,16 +603,16 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, compilePattern = joinPath(conf.cCompilerPath, exe) else: - compilePattern = getCompilerExe(conf, c, cfile.cname) + compilePattern = exe includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)])) - var cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string) + let cf = if noAbsolutePaths(conf): AbsoluteFile extractFilename(cfile.cname.string) else: cfile.cname - var objfile = + let objfile = if cfile.obj.isEmpty: - if not cfile.flags.contains(CfileFlag.External) or noAbsolutePaths(conf): + if CfileFlag.External notin cfile.flags or noAbsolutePaths(conf): toObjFile(conf, cf).string else: completeCfilePath(conf, toObjFile(conf, cf)).string @@ -640,7 +630,8 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, "dfile", dfile, "file", cfsh, "objfile", quoteShell(objfile), "options", options, "include", includeCmd, "nim", getPrefixDir(conf).string, - "lib", conf.libpath.string]) + "lib", conf.libpath.string, + "ccenvflags", envFlags(conf)]) if optProduceAsm in conf.globalOptions: if CC[conf.cCompiler].produceAsm.len > 0: @@ -654,13 +645,14 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile, "for the selected C compiler: " & CC[conf.cCompiler].name) result.add(' ') - result.addf(CC[c].compileTmpl, [ + strutils.addf(result, CC[c].compileTmpl, [ "dfile", dfile, "file", cfsh, "objfile", quoteShell(objfile), "options", options, "include", includeCmd, "nim", quoteShell(getPrefixDir(conf)), "lib", quoteShell(conf.libpath), - "vccplatform", vccplatform(conf)]) + "vccplatform", vccplatform(conf), + "ccenvflags", envFlags(conf)]) proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash = result = secureHash( @@ -671,12 +663,11 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash = getCompileCFileCmd(conf, cfile)) proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = - if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM, cmdNone}: - return false + if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good - var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1") - var currentHash = footprint(conf, cfile) - var f: File + let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1") + let currentHash = footprint(conf, cfile) + var f: File = default(File) if open(f, hashFile.string, fmRead): let oldHash = parseSecureHash(f.readLine()) close(f) @@ -689,8 +680,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool = close(f) proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) = + # we want to generate the hash file unconditionally + let extFileChanged = externalFileChanged(conf, c) if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and - not externalFileChanged(conf, c): + not extFileChanged: c.flags.incl CfileFlag.Cached else: # make sure Nim keeps recompiling the external file on reruns @@ -704,34 +697,14 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) = flags: {CfileFlag.External}) addExternalFileToCompile(conf, c) -proc compileCFiles(conf: ConfigRef; list: CfileList, script: var Rope, cmds: var TStringSeq, - prettyCmds: var TStringSeq) = - var currIdx = 0 - for it in list: - # call the C compiler for the .c file: - if it.flags.contains(CfileFlag.Cached): continue - var compileCmd = getCompileCFileCmd(conf, it, currIdx == list.len - 1, produceOutput=true) - inc currIdx - if optCompileOnly notin conf.globalOptions: - cmds.add(compileCmd) - let (_, name, _) = splitFile(it.cname) - prettyCmds.add(if hintCC in conf.notes: "CC: " & demanglePackageName(name) else: "") - if optGenScript in conf.globalOptions: - script.add(compileCmd) - script.add("\n") - proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, - objfiles: string, isDllBuild: bool): string = + objfiles: string, isDllBuild: bool, removeStaticFile: bool): string = if optGenStaticLib in conf.globalOptions: - var libname: string - if not conf.outFile.isEmpty: - libname = conf.outFile.string.expandTilde - if not libname.isAbsolute(): - libname = getCurrentDir() / libname - else: - libname = (libNameTmpl(conf) % splitFile(conf.projectName).name) - result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname), - "objfiles", objfiles] + if removeStaticFile: + removeFile output # fixes: bug #16947 + result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output), + "objfiles", objfiles, + "vccplatform", vccplatform(conf)] else: var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe") if linkerExe.len == 0: linkerExe = getLinkerExe(conf, conf.cCompiler) @@ -764,7 +737,7 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, "exefile", exefile, "nim", getPrefixDir(conf).string, "lib", conf.libpath.string]) result.add ' ' - result.addf(linkTmpl, ["builddll", builddll, + strutils.addf(result, linkTmpl, ["builddll", builddll, "mapfile", mapfile, "buildgui", buildgui, "options", linkOptions, "objfiles", objfiles, "exefile", exefile, @@ -812,8 +785,9 @@ proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile, if optCDebug in conf.globalOptions and conf.cCompiler == ccVcc: result.add " /Zi /FS /Od" -template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string): string = - getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions) +template getLinkCmd(conf: ConfigRef; output: AbsoluteFile, objfiles: string, + removeStaticFile = false): string = + getLinkCmd(conf, output, objfiles, optGenDynLib in conf.globalOptions, removeStaticFile) template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body: untyped) = try: @@ -827,26 +801,23 @@ template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body (ose.msg & " " & $ose.errorCode)) raise +proc getExtraCmds(conf: ConfigRef; output: AbsoluteFile): seq[string] = + result = @[] + when defined(macosx): + if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions: + # if needed, add an option to skip or override location + result.add "dsymutil " & $(output).quoteShell + proc execLinkCmd(conf: ConfigRef; linkCmd: string) = tryExceptOSErrorMessage(conf, "invocation of external linker program failed."): execExternalProgram(conf, linkCmd, hintLinking) -proc maybeRunDsymutil(conf: ConfigRef; exe: AbsoluteFile) = - when defined(osx): - if optCDebug notin conf.globalOptions: return - # if needed, add an option to skip or override location - let cmd = "dsymutil " & $(exe).quoteShell - conf.extraCmds.add cmd - tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."): - execExternalProgram(conf, cmd, hintExecuting) - proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) = let runCb = proc (idx: int, p: Process) = let exitCode = p.peekExitCode if exitCode != 0: rawMessage(conf, errGenerated, "execution of an external compiler program '" & - cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n" & - p.outputStream.readAll.strip) + cmds[idx] & "' failed with exit code: " & $exitCode & "\n\n") if conf.numberOfProcessors == 0: conf.numberOfProcessors = countProcessors() var res = 0 if conf.numberOfProcessors <= 1: @@ -858,15 +829,8 @@ proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: cmds[i]) else: tryExceptOSErrorMessage(conf, "invocation of external compiler program failed."): - if optListCmd in conf.globalOptions or conf.verbosity > 1: - res = execProcesses(cmds, {poEchoCmd, poStdErrToStdOut, poUsePath}, - conf.numberOfProcessors, afterRunEvent=runCb) - elif conf.verbosity == 1: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath}, + res = execProcesses(cmds, {poStdErrToStdOut, poUsePath, poParentStreams}, conf.numberOfProcessors, prettyCb, afterRunEvent=runCb) - else: - res = execProcesses(cmds, {poStdErrToStdOut, poUsePath}, - conf.numberOfProcessors, afterRunEvent=runCb) if res != 0: if conf.numberOfProcessors <= 1: rawMessage(conf, errGenerated, "execution of an external program failed: '$1'" % @@ -894,10 +858,22 @@ proc linkViaResponseFile(conf: ConfigRef; cmd: string) = else: writeFile(linkerArgs, args) try: - execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs) + when defined(macosx): + execLinkCmd(conf, "xargs " & cmd.substr(0, last) & " < " & linkerArgs) + else: + execLinkCmd(conf, cmd.substr(0, last) & " @" & linkerArgs) finally: removeFile(linkerArgs) +proc linkViaShellScript(conf: ConfigRef; cmd: string) = + let linkerScript = conf.projectName & "_" & "linkerScript.sh" + writeFile(linkerScript, cmd) + let shell = getEnv("SHELL") + try: + execLinkCmd(conf, shell & " " & linkerScript) + finally: + removeFile(linkerScript) + proc getObjFilePath(conf: ConfigRef, f: Cfile): string = if noAbsolutePaths(conf): f.obj.extractFilename else: f.obj.string @@ -908,21 +884,52 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu else: platform.OS[conf.target.targetOS].dllFrmt % basename result = conf.getNimcacheDir / RelativeFile(targetName) +proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string = + result = "" + if conf.hasHint(hintCC): + if optListCmd in conf.globalOptions or conf.verbosity > 1: + result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd) + else: + result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name) + +proc preventLinkCmdMaxCmdLen(conf: ConfigRef, linkCmd: string) = + # Prevent linkcmd from exceeding the maximum command line length. + # Windows's command line limit is about 8K (8191 characters) so C compilers on + # Windows support a feature where the command line can be passed via ``@linkcmd`` + # to them. + const MaxCmdLen = when defined(windows): 8_000 elif defined(macosx): 260_000 else: 32_000 + if linkCmd.len > MaxCmdLen: + when defined(macosx): + linkViaShellScript(conf, linkCmd) + else: + linkViaResponseFile(conf, linkCmd) + else: + execLinkCmd(conf, linkCmd) + proc callCCompiler*(conf: ConfigRef) = var - linkCmd: string + linkCmd: string = "" + extraCmds: seq[string] if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}: return # speed up that call if only compiling and no script shall be # generated #var c = cCompiler - var script: Rope = nil - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] - let prettyCb = proc (idx: int) = - when declared(echo): - let cmd = prettyCmds[idx] - if cmd != "": echo cmd - compileCFiles(conf, conf.toCompile, script, cmds, prettyCmds) + var script: Rope = "" + var cmds: TStringSeq = default(TStringSeq) + var prettyCmds: TStringSeq = default(TStringSeq) + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + + for idx, it in conf.toCompile: + # call the C compiler for the .c file: + if CfileFlag.Cached in it.flags: continue + let compileCmd = getCompileCFileCmd(conf, it, idx == conf.toCompile.len - 1, produceOutput=true) + if optCompileOnly notin conf.globalOptions: + cmds.add(compileCmd) + prettyCmds.add displayProgressCC(conf, $it.cname, compileCmd) + if optGenScript in conf.globalOptions: + script.add(compileCmd) + script.add("\n") + if optCompileOnly notin conf.globalOptions: execCmdsInParallel(conf, cmds, prettyCb) if optNoLinking notin conf.globalOptions: @@ -941,14 +948,14 @@ proc callCCompiler*(conf: ConfigRef) = # don't relink each of the many binaries (one for each source file) if the nim code is # cached because that would take too much time for small changes - the only downside to # this is that if an external-to-link file changes the final target wouldn't be relinked - if x.flags.contains(CfileFlag.Cached): continue + if CfileFlag.Cached in x.flags: continue # we pass each object file as if it is the project file - a .dll will be created for each such # object file in the nimcache directory, and only in the case of the main project file will # there be probably an executable (if the project is such) which will be copied out of the nimcache let objFile = conf.getObjFilePath(x) let buildDll = idx != mainFileIdx let linkTarget = conf.hcrLinkTargetName(objFile, not buildDll) - cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll)) + cmds.add(getLinkCmd(conf, linkTarget, objfiles & " " & quoteShell(objFile), buildDll, removeStaticFile = true)) # try to remove all .pdb files for the current binary so they don't accumulate endlessly in the nimcache # for more info check the comment inside of getLinkCmd() where the /PDB:<filename> MSVC flag is used if isVSCompatible(conf): @@ -959,10 +966,10 @@ proc callCCompiler*(conf: ConfigRef) = prettyCmds = map(prettyCmds, proc (curr: string): string = return curr.replace("CC", "Link")) execCmdsInParallel(conf, cmds, prettyCb) # only if not cached - copy the resulting main file from the nimcache folder to its originally intended destination - if not conf.toCompile[mainFileIdx].flags.contains(CfileFlag.Cached): + if CfileFlag.Cached notin conf.toCompile[mainFileIdx].flags: let mainObjFile = getObjFilePath(conf, conf.toCompile[mainFileIdx]) - var src = conf.hcrLinkTargetName(mainObjFile, true) - var dst = conf.prepareToWriteOutput + let src = conf.hcrLinkTargetName(mainObjFile, true) + let dst = conf.prepareToWriteOutput copyFileWithPermissions(src.string, dst.string) else: for x in conf.toCompile: @@ -970,18 +977,14 @@ proc callCCompiler*(conf: ConfigRef) = objfiles.add(' ') objfiles.add(quoteShell(objFile)) let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput - else: AbsoluteFile(conf.projectName) - linkCmd = getLinkCmd(conf, mainOutput, objfiles) + else: AbsoluteFile(conf.outFile) + + linkCmd = getLinkCmd(conf, mainOutput, objfiles, removeStaticFile = true) + extraCmds = getExtraCmds(conf, mainOutput) if optCompileOnly notin conf.globalOptions: - const MaxCmdLen = when defined(windows): 8_000 else: 32_000 - if linkCmd.len > MaxCmdLen: - # Windows's command line limit is about 8K (don't laugh...) so C compilers on - # Windows support a feature where the command line can be passed via ``@linkcmd`` - # to them. - linkViaResponseFile(conf, linkCmd) - else: - execLinkCmd(conf, linkCmd) - maybeRunDsymutil(conf, mainOutput) + preventLinkCmdMaxCmdLen(conf, linkCmd) + for cmd in extraCmds: + execExternalProgram(conf, cmd, hintExecuting) else: linkCmd = "" if optGenScript in conf.globalOptions: @@ -989,168 +992,113 @@ proc callCCompiler*(conf: ConfigRef) = script.add("\n") generateScript(conf, script) -#from json import escapeJson -import json, std / sha1 - template hashNimExe(): string = $secureHashFile(os.getAppFilename()) -proc writeJsonBuildInstructions*(conf: ConfigRef) = - template lit(x: untyped) = f.write x - template str(x: untyped) = - when compiles(escapeJson(x, buf)): - buf.setLen 0 - escapeJson(x, buf) - f.write buf - else: - f.write escapeJson(x) - - proc cfiles(conf: ConfigRef; f: File; buf: var string; clist: CfileList, isExternal: bool) = - var i = 0 - for it in clist: - if CfileFlag.Cached in it.flags: continue - let compileCmd = getCompileCFileCmd(conf, it) - if i > 0: lit ",\L" - lit "[" - str it.cname.string - lit ", " - str compileCmd - lit "]" - inc i - - proc linkfiles(conf: ConfigRef; f: File; buf, objfiles: var string; clist: CfileList; - llist: seq[string]) = - var pastStart = false - for it in llist: - let objfile = if noAbsolutePaths(conf): it.extractFilename - else: it - let objstr = addFileExt(objfile, CC[conf.cCompiler].objExt) - objfiles.add(' ') - objfiles.add(objstr) - if pastStart: lit ",\L" - str objstr - pastStart = true - - for it in clist: - let objstr = quoteShell(it.obj) - objfiles.add(' ') - objfiles.add(objstr) - if pastStart: lit ",\L" - str objstr - pastStart = true - lit "\L" - - proc depfiles(conf: ConfigRef; f: File) = - var i = 0 +proc jsonBuildInstructionsFile*(conf: ConfigRef): AbsoluteFile = + # `outFile` is better than `projectName`, as it allows having different json + # files for a given source file compiled with different options; it also + # works out of the box with `hashMainCompilationParams`. + result = getNimcacheDir(conf) / conf.outFile.changeFileExt("json") + +const cacheVersion = "D20240927T193831" # update when `BuildCache` spec changes +type BuildCache = object + cacheVersion: string + outputFile: string + outputLastModificationTime: string + compile: seq[(string, string)] + link: seq[string] + linkcmd: string + extraCmds: seq[string] + configFiles: seq[string] # the hash shouldn't be needed + stdinInput: bool + projectIsCmd: bool + cmdInput: string + currentDir: string + cmdline: string + depfiles: seq[(string, string)] + nimexe: string + +proc writeJsonBuildInstructions*(conf: ConfigRef; deps: StringTableRef) = + var linkFiles = collect(for it in conf.externalToLink: + var it = it + if conf.noAbsolutePaths: it = it.extractFilename + it.addFileExt(CC[conf.cCompiler].objExt)) + for it in conf.toCompile: linkFiles.add it.obj.string + var bcache = BuildCache( + cacheVersion: cacheVersion, + outputFile: conf.absOutFile.string, + compile: collect(for i, it in conf.toCompile: + if CfileFlag.Cached notin it.flags: (it.cname.string, getCompileCFileCmd(conf, it))), + link: linkFiles, + linkcmd: getLinkCmd(conf, conf.absOutFile, linkFiles.quoteShellCommand), + extraCmds: getExtraCmds(conf, conf.absOutFile), + stdinInput: conf.projectIsStdin, + projectIsCmd: conf.projectIsCmd, + cmdInput: conf.cmdInput, + configFiles: conf.configFiles.mapIt(it.string), + currentDir: getCurrentDir()) + if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): + bcache.cmdline = conf.commandLine for it in conf.m.fileInfos: let path = it.fullPath.string if isAbsolute(path): # TODO: else? - if i > 0: lit "],\L" - lit "[" - str path - lit ", " - str $secureHashFile(path) - inc i - lit "]\L" - - - var buf = newStringOfCap(50) - - let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json") - - var f: File - if open(f, jsonFile.string, fmWrite): - lit "{\"compile\":[\L" - cfiles(conf, f, buf, conf.toCompile, false) - lit "],\L\"link\":[\L" - var objfiles = "" - # XXX add every file here that is to link - linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink) - - lit "],\L\"linkcmd\": " - str getLinkCmd(conf, conf.absOutFile, objfiles) - - lit ",\L\"extraCmds\": " - lit $(%* conf.extraCmds) - - if optRun in conf.globalOptions or isDefined(conf, "nimBetterRun"): - lit ",\L\"cmdline\": " - str conf.commandLine - lit ",\L\"depfiles\":[\L" - depfiles(conf, f) - lit "],\L\"nimexe\": \L" - str hashNimExe() - lit "\L" - - lit "\L}\L" - close(f) - -proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile): bool = - let jsonFile = toGeneratedFile(conf, projectfile, "json") - if not fileExists(jsonFile): return true - if not fileExists(conf.absOutFile): return true + if path in deps: + bcache.depfiles.add (path, deps[path]) + else: # backup for configs etc. + bcache.depfiles.add (path, $secureHashFile(path)) + + bcache.nimexe = hashNimExe() + if fileExists(bcache.outputFile): + bcache.outputLastModificationTime = $getLastModificationTime(bcache.outputFile) + conf.jsonBuildFile = conf.jsonBuildInstructionsFile + conf.jsonBuildFile.string.writeFile(bcache.toJson.pretty) + +proc changeDetectedViaJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile): bool = result = false - try: - let data = json.parseFile(jsonFile.string) - if not data.hasKey("depfiles") or not data.hasKey("cmdline"): - return true - let oldCmdLine = data["cmdline"].getStr - if conf.commandLine != oldCmdLine: - return true - if hashNimExe() != data["nimexe"].getStr: - return true - let depfilesPairs = data["depfiles"] - doAssert depfilesPairs.kind == JArray - for p in depfilesPairs: - doAssert p.kind == JArray - # >= 2 for forwards compatibility with potential later .json files: - doAssert p.len >= 2 - let depFilename = p[0].getStr - let oldHashValue = p[1].getStr - let newHashValue = $secureHashFile(depFilename) - if oldHashValue != newHashValue: - return true + if not fileExists(jsonFile) or not fileExists(conf.absOutFile): return true + var bcache: BuildCache = default(BuildCache) + try: bcache.fromJson(jsonFile.string.parseFile) except IOError, OSError, ValueError: - echo "Warning: JSON processing failed: ", getCurrentExceptionMsg() - result = true - -proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) = - let jsonFile = toGeneratedFile(conf, projectfile, "json") - try: - let data = json.parseFile(jsonFile.string) - let toCompile = data["compile"] - doAssert toCompile.kind == JArray - var cmds: TStringSeq = @[] - var prettyCmds: TStringSeq = @[] - for c in toCompile: - doAssert c.kind == JArray - doAssert c.len >= 2 - - cmds.add(c[1].getStr) - let (_, name, _) = splitFile(c[0].getStr) - prettyCmds.add("CC: " & demanglePackageName(name)) - - let prettyCb = proc (idx: int) = - when declared(echo): - echo prettyCmds[idx] - execCmdsInParallel(conf, cmds, prettyCb) - - let linkCmd = data["linkcmd"] - doAssert linkCmd.kind == JString - execLinkCmd(conf, linkCmd.getStr) - if data.hasKey("extraCmds"): - let extraCmds = data["extraCmds"] - doAssert extraCmds.kind == JArray - for cmd in extraCmds: - doAssert cmd.kind == JString, $cmd.kind - let cmd2 = cmd.getStr - execExternalProgram(conf, cmd2, hintExecuting) - - except: - when declared(echo): - echo getCurrentException().getStackTrace() - quit "error evaluating JSON file: " & jsonFile.string + stderr.write "Warning: JSON processing failed for: $#\n" % jsonFile.string + return true + if bcache.currentDir != getCurrentDir() or # fixes bug #16271 + bcache.configFiles != conf.configFiles.mapIt(it.string) or + bcache.cacheVersion != cacheVersion or bcache.outputFile != conf.absOutFile.string or + bcache.cmdline != conf.commandLine or bcache.nimexe != hashNimExe() or + bcache.projectIsCmd != conf.projectIsCmd or conf.cmdInput != bcache.cmdInput: return true + if bcache.stdinInput or conf.projectIsStdin: return true + # xxx optimize by returning false if stdin input was the same + for (file, hash) in bcache.depfiles: + if $secureHashFile(file) != hash: return true + if bcache.outputLastModificationTime != $getLastModificationTime(bcache.outputFile): + return true + +proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) = + var bcache: BuildCache = default(BuildCache) + try: bcache.fromJson(jsonFile.string.parseFile) + except ValueError, KeyError, JsonKindError: + let e = getCurrentException() + conf.quitOrRaise "\ncaught exception:\n$#\nstacktrace:\n$#error evaluating JSON file: $#" % + [e.msg, e.getStackTrace(), jsonFile.string] + let output = bcache.outputFile + createDir output.parentDir + let outputCurrent = $conf.absOutFile + if output != outputCurrent or bcache.cacheVersion != cacheVersion: + globalError(conf, gCmdLineInfo, + "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % + [outputCurrent, output, jsonFile.string]) + var cmds: TStringSeq = default(TStringSeq) + var prettyCmds: TStringSeq = default(TStringSeq) + let prettyCb = proc (idx: int) = writePrettyCmdsStderr(prettyCmds[idx]) + for (name, cmd) in bcache.compile: + cmds.add cmd + prettyCmds.add displayProgressCC(conf, name, cmd) + execCmdsInParallel(conf, cmds, prettyCb) + preventLinkCmdMaxCmdLen(conf, bcache.linkcmd) + for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting) proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope = + result = "" for it in list: result.addf("--file:r\"$1\"$N", [rope(it.cname.string)]) |