diff options
Diffstat (limited to 'tools')
40 files changed, 3844 insertions, 2716 deletions
diff --git a/tools/atlas/parse_requires.nim b/tools/atlas/parse_requires.nim new file mode 100644 index 000000000..66879d04f --- /dev/null +++ b/tools/atlas/parse_requires.nim @@ -0,0 +1,101 @@ +## Utility API for Nim package managers. +## (c) 2021 Andreas Rumpf + +import std / strutils +import ".." / ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils] + +type + NimbleFileInfo* = object + requires*: seq[string] + srcDir*: string + tasks*: seq[(string, string)] + +proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = + case n.kind + of nkStmtList, nkStmtListExpr: + for child in n: + extract(child, conf, result) + of nkCallKinds: + if n[0].kind == nkIdent: + case n[0].ident.s + of "requires": + for i in 1..<n.len: + var ch = n[i] + while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0: ch = ch.lastSon + if ch.kind in {nkStrLit..nkTripleStrLit}: + result.requires.add ch.strVal + else: + localError(conf, ch.info, "'requires' takes string literals") + of "task": + if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}: + result.tasks.add((n[1].ident.s, n[2].strVal)) + else: discard + of nkAsgn, nkFastAsgn: + if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0: + if n[1].kind in {nkStrLit..nkTripleStrLit}: + result.srcDir = n[1].strVal + else: + localError(conf, n[1].info, "assignments to 'srcDir' must be string literals") + else: + discard + +proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo = + ## Extract the `requires` information from a Nimble file. This does **not** + ## evaluate the Nimble file. Errors are produced on stderr/stdout and are + ## formatted as the Nim compiler does it. The parser uses the Nim compiler + ## as an API. The result can be empty, this is not an error, only parsing + ## errors are reported. + var conf = newConfigRef() + conf.foreignPackageNotes = {} + conf.notes = {} + conf.mainPackageNotes = {} + + let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + extract(parseAll(parser), conf, result) + closeParser(parser) + +const Operators* = {'<', '>', '=', '&', '@', '!', '^'} + +proc token(s: string; idx: int; lit: var string): int = + var i = idx + if i >= s.len: return i + while s[i] in Whitespace: inc(i) + case s[i] + of Letters, '#': + lit.add s[i] + inc i + while i < s.len and s[i] notin (Whitespace + {'@', '#'}): + lit.add s[i] + inc i + of '0'..'9': + while i < s.len and s[i] in {'0'..'9', '.'}: + lit.add s[i] + inc i + of '"': + inc i + while i < s.len and s[i] != '"': + lit.add s[i] + inc i + inc i + of Operators: + while i < s.len and s[i] in Operators: + lit.add s[i] + inc i + else: + lit.add s[i] + inc i + result = i + +iterator tokenizeRequires*(s: string): string = + var start = 0 + var tok = "" + while start < s.len: + tok.setLen 0 + start = token(s, start, tok) + yield tok + +when isMainModule: + for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"): + echo x diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index bfed8d88f..46bc39470 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -1,6 +1,6 @@ ##[ avoid code duplication in CI pipelines. -For now, this is only used for openbsd, but there is a lot of other code +For now, this is only used for openbsd + freebsd, but there is a lot of other code duplication that could be removed. ## usage @@ -10,22 +10,15 @@ nim r tools/ci_generate.nim ``` ]## -import std/strformat +import std/[strformat, os] -proc genCIopenbsd(batch: int, num: int): string = +const doNotEdit = "DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`" +proc genCiBsd(header: string, batch: int, num: int): string = result = fmt""" -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` +## {doNotEdit} + +{header} -image: openbsd/latest -packages: -- gmake -- sqlite3 -- node -- boehm-gc -- pcre -- sfml -- sdl2 -- libffi sources: - https://github.com/nim-lang/Nim environment: @@ -33,31 +26,116 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch + . ci/funs.sh && nimBuildCsourcesIfNeeded echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim - if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure to: Andreas Rumpf <rumpf_a@web.de> """ +proc genBuildExtras(echoRun, koch, nim: string): string = + result = fmt""" +{echoRun} {nim} c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch +{echoRun} {koch} boot -d:release --skipUserCfg --skipParentCfg --hints:off +{echoRun} {koch} tools --skipUserCfg --skipParentCfg --hints:off +""" + +proc genWindowsScript(buildAll: bool): string = + result = fmt""" +@echo off +rem {doNotEdit} +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) +""" + + if buildAll: + result.add genBuildExtras("", "koch", r"bin\nim.exe") + +proc genPosixScript(): string = + result = fmt""" +#!/bin/sh +# {doNotEdit} + +# build development version of the compiler; can be rerun safely. +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin + +set -u # error on undefined variables +set -e # exit on first error + +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" + +{genBuildExtras("echo_run", "./koch", "bin/nim")} +""" + proc main()= - let num = 3 + let dir = ".builds" + # not too large to be resource friendly, refs bug #17107 + let num = 2 # if you reduce this, make sure to remove files that shouldn't be generated, - # or better, do the cleanup logic here eg: `rm .builds/openbsd_*` + # or better, do the cleanup logic here e.g.: `rm .builds/openbsd_*` + let headerFreebsd = """ +# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd +image: freebsd/latest +packages: +- databases/sqlite3 +- devel/boehm-gc-threaded +- devel/pcre +- devel/sdl20 +- devel/sfml +- www/node +- devel/gmake +""" + + let headerOpenbsd = """ +image: openbsd/latest +packages: +- gmake +- sqlite3 +- node +- boehm-gc +- pcre +- sfml +- sdl2 +- libffi +""" + for i in 0..<num: - let file = fmt".builds/openbsd_{i}.yml" - let code = genCIopenbsd(i, num) - writeFile(file, code) + writeFile(dir / fmt"openbsd_{i}.yml", genCiBsd(headerOpenbsd, i, num)) + writeFile(dir / "freebsd.yml", genCiBsd(headerFreebsd, 0, 1)) + writeFile("build_all.sh", genPosixScript()) + writeFile("build_all.bat", genWindowsScript(buildAll = true)) + writeFile("ci/build_autogen.bat", genWindowsScript(buildAll = false)) when isMainModule: main() diff --git a/tools/compiler.gdb b/tools/compiler.gdb new file mode 100644 index 000000000..c81f47152 --- /dev/null +++ b/tools/compiler.gdb @@ -0,0 +1,39 @@ +# create a breakpoint on `debugutils.enteringDebugSection` +define enable_enteringDebugSection + break -function enteringDebugSection + # run these commands once breakpoint enteringDebugSection is hit + command + # enable all breakpoints and watchpoints + enable + # continue execution + cont + end +end + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +define enable_exitingDebugSection + break -function exitingDebugSection + # run these commands once breakpoint exitingDebugSection is hit + command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # continue execution + cont + end +end + +# some commands can't be set until the process is running, so set an entry breakpoint +break -function NimMain +# run these commands once breakpoint NimMain is hit +command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # no longer need this breakpoint + delete -function NimMain + # continue execution + cont +end diff --git a/tools/compiler.lldb b/tools/compiler.lldb new file mode 100644 index 000000000..e0b375055 --- /dev/null +++ b/tools/compiler.lldb @@ -0,0 +1,40 @@ +# create a breakpoint on `debugutils.enteringDebugSection` named enteringDebugSection +breakpoint set -n 'enteringDebugSection' -N enteringDebugSection +# run these commands once breakpoint enteringDebugSection is hit +breakpoint command add enteringDebugSection + # enable all breakpoints + breakpoint enable + # enable all watchpoints + # watchpoint enable # FIXME: not currently working for unknown reason + # continue execution + continue +DONE + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +breakpoint set -n 'exitingDebugSection' -N exitingDebugSection +# run these commands once breakpoint exitingDebugSection is hit +breakpoint command add exitingDebugSection + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + breakpoint enable enteringDebugSection + # continue execution + continue +DONE + +# some commands can't be set until the process is running, so set an entry breakpoint +breakpoint set -n NimMain -N NimMain +# run these commands once breakpoint NimMain is hit +breakpoint command add NimMain + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + # enable the enteringDebugSection breakpoint though + breakpoint enable enteringDebugSection + # no longer need this breakpoint + breakpoint delete NimMain + # continue execution + continue +DONE diff --git a/tools/debug/customdebugtype.nim b/tools/debug/customdebugtype.nim new file mode 100644 index 000000000..f48979661 --- /dev/null +++ b/tools/debug/customdebugtype.nim @@ -0,0 +1,72 @@ +## This is a demo file containing an example of how to +## create custom LLDB summaries and objects with synthetic +## children. These are implemented in Nim and called from the Python +## nimlldb.py module. +## +## For summaries, prefix your proc names with "lldbDebugSummary", use +## the `{.exportc.}` pragma, and return a string. Also, any `$` proc +## that is available will be used for a given type. +## +## For creating a synthetic object (LLDB will display the children), use +## the prefix "lldbDebugSynthetic", use the `{.exportc.}` pragma, and +## return any Nim object, array, or sequence. Returning a Nim object +## will display the fields and values of the object as children. +## Returning an array or sequence will display children with the index +## surrounded by square brackets as the key name +## +## You may also return a Nim table that contains the string +## "LLDBDynamicObject" (case insensitive). This allows for dynamic +## fields to be created at runtime instead of at compile time if you +## return a Nim object as mentioned above. See the proc +## `lldbDebugSyntheticDynamicFields` below for an example + +import intsets +import tables + +type + CustomType* = object of RootObj # RootObj is not necessary, but can be used + myField*: int + + DynamicFields* = object + customField*: string + + CustomSyntheticReturn* = object + differentField*: float + + LLDBDynamicObject = object + fields: TableRef[string, int] + + LLDBDynamicObjectDynamicFields = object + fields: TableRef[string, string] + +proc lldbDebugSummaryCustomType*(ty: CustomType): string {.exportc.} = + ## Will display "CustomType(myField: <int_val>)" as a summary + result = "CustomType" & $ty + +proc lldbDebugSyntheticCustomType*(ty: CustomType): CustomSyntheticReturn {.exportc.} = + ## Will display differentField: <float_val> as a child of CustomType instead of + ## myField: <int_val> + result = CustomSyntheticReturn(differentField: ty.myField.float) + +proc lldbDebugSyntheticDynamicFields*(ty: DynamicFields): LLDBDynamicObjectDynamicFields {.exportc.} = + ## Returning an object that contains "LLDBDynamicObject" in the type name will expect an + ## object with one property that is a Nim Table/TableRef. If the key is a string, + ## it will appear in the debugger like an object field name. The value will be whatever you + ## set it to here as well. + let fields = {"customFieldName": ty.customField & " MORE TEXT"}.newTable() + return LLDBDynamicObjectDynamicFields(fields: fields) + +proc lldbDebugSummaryIntSet*(intset: IntSet): string {.exportc.} = + ## This will print the object in the LLDB summary just as Nim prints it + result = $intset + +proc lldbDebugSyntheticIntSet*(intset: IntSet): seq[int] {.exportc.} = + ## This will create a synthetic object to make it so that IntSet + ## will appear as a Nim object in the LLDB debugger window + ## + ## returning a seq here will display children like: + ## [0]: <child_value> + ## + result = newSeqOfCap[int](intset.len) + for val in intset: + result.add(val) diff --git a/tools/nim-gdb.py b/tools/debug/nim-gdb.py index e994531b6..8c9854bda 100644 --- a/tools/nim-gdb.py +++ b/tools/debug/nim-gdb.py @@ -1,6 +1,7 @@ import gdb import re import sys +import traceback # some feedback that the nim runtime support is loading, isn't a bad # thing at all. @@ -13,14 +14,23 @@ def printErrorOnce(id, message): global errorSet if id not in errorSet: errorSet.add(id) - gdb.write(message, gdb.STDERR) + gdb.write("printErrorOnce: " + message, gdb.STDERR) +def debugPrint(x): + gdb.write(str(x) + "\n", gdb.STDERR) + +NIM_STRING_TYPES = ["NimStringDesc", "NimStringV2"] ################################################################################ ##### Type pretty printers ################################################################################ -type_hash_regex = re.compile("^\w*_([A-Za-z0-9]*)$") +type_hash_regex = re.compile("^([A-Za-z0-9]*)_([A-Za-z0-9]*)_+([A-Za-z0-9]*)$") + +def getNimName(typ): + if m := type_hash_regex.match(typ): + return m.group(2) + return f"unknown <{typ}>" def getNimRti(type_name): """ Return a ``gdb.Value`` object for the Nim Runtime Information of ``type_name``. """ @@ -29,10 +39,25 @@ def getNimRti(type_name): # every non trivial Nim type. m = type_hash_regex.match(type_name) if m: - try: - return gdb.parse_and_eval("NTI__" + m.group(1) + "_") - except: - return None + lookups = [ + "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", + "NTI" + "__" + m.group(3) + "_", + "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" + ] + for l in lookups: + try: + return gdb.parse_and_eval(l) + except: + pass + None + +def getNameFromNimRti(rti): + """ Return name (or None) given a Nim RTI ``gdb.Value`` """ + try: + # sometimes there isn't a name field -- example enums + return rti['name'].string(encoding="utf-8", errors="ignore") + except: + return None class NimTypeRecognizer: # this type map maps from types that are generated in the C files to @@ -42,30 +67,25 @@ class NimTypeRecognizer: # ``int``. type_map_static = { - 'NI': 'system.int', 'NI8': 'int8', 'NI16': 'int16', 'NI32': 'int32', 'NI64': 'int64', - 'NU': 'uint', 'NU8': 'uint8','NU16': 'uint16', 'NU32': 'uint32', 'NU64': 'uint64', + 'NI': 'system.int', 'NI8': 'int8', 'NI16': 'int16', 'NI32': 'int32', + 'NI64': 'int64', + + 'NU': 'uint', 'NU8': 'uint8','NU16': 'uint16', 'NU32': 'uint32', + 'NU64': 'uint64', + 'NF': 'float', 'NF32': 'float32', 'NF64': 'float64', - 'NIM_BOOL': 'bool', 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', - 'NimStringDesc': 'string' + + 'NIM_BOOL': 'bool', + + 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string', 'NimStringV2': 'string' } - # Normally gdb distinguishes between the command `ptype` and - # `whatis`. `ptype` prints a very detailed view of the type, and - # `whatis` a very brief representation of the type. I haven't - # figured out a way to know from the type printer that is - # implemented here how to know if a type printer should print the - # short representation or the long representation. As a hacky - # workaround I just say I am not resposible for printing pointer - # types (seq and string are exception as they are semantically - # values). this way the default type printer will handle pointer - # types and dive into the members of that type. So I can still - # control with `ptype myval` and `ptype *myval` if I want to have - # detail or not. I this this method stinks but I could not figure - # out a better solution. - - object_type_pattern = re.compile("^(\w*):ObjectType$") + # object_type_pattern = re.compile("^(\w*):ObjectType$") def recognize(self, type_obj): + # skip things we can't handle like functions + if type_obj.code in [gdb.TYPE_CODE_FUNC, gdb.TYPE_CODE_VOID]: + return None tname = None if type_obj.tag is not None: @@ -75,44 +95,49 @@ class NimTypeRecognizer: # handle pointer types if not tname: - if type_obj.code == gdb.TYPE_CODE_PTR: + target_type = type_obj + if type_obj.code in [gdb.TYPE_CODE_PTR]: target_type = type_obj.target() - target_type_name = target_type.name - if target_type_name: - # visualize 'string' as non pointer type (unpack pointer type). - if target_type_name == "NimStringDesc": - tname = target_type_name # could also just return 'string' - # visualize 'seq[T]' as non pointer type. - if target_type_name.find('tySequence_') == 0: - tname = target_type_name - - if not tname: - # We are not resposible for this type printing. - # Basically this means we don't print pointer types. - return None - result = self.type_map_static.get(tname, None) - if result: - return result + if target_type.name: + # visualize 'string' as non pointer type (unpack pointer type). + if target_type.name == "NimStringDesc": + tname = target_type.name # could also just return 'string' + else: + rti = getNimRti(target_type.name) + if rti: + return getNameFromNimRti(rti) + + if tname: + result = self.type_map_static.get(tname, None) + if result: + return result + elif tname.startswith("tyEnum_"): + return getNimName(tname) + elif tname.startswith("tyTuple__"): + # We make the name be the field types (Just like in Nim) + fields = ", ".join([self.recognize(field.type) for field in type_obj.fields()]) + return f"({fields})" + + rti = getNimRti(tname) + if rti: + return getNameFromNimRti(rti) - rti = getNimRti(tname) - if rti: - return rti['name'].string("utf-8", "ignore") - else: - return None + return None class NimTypePrinter: """Nim type printer. One printer for all Nim types.""" - # enabling and disabling of type printers can be done with the # following gdb commands: # # enable type-printer NimTypePrinter # disable type-printer NimTypePrinter + # relevant docs: https://sourceware.org/gdb/onlinedocs/gdb/Type-Printing-API.html name = "NimTypePrinter" - def __init__ (self): + + def __init__(self): self.enabled = True def instantiate(self): @@ -126,7 +151,7 @@ class DollarPrintFunction (gdb.Function): "Nim's equivalent of $ operator as a gdb function, available in expressions `print $dollar(myvalue)" dollar_functions = re.findall( - 'NimStringDesc \*(dollar__[A-z0-9_]+?)\(([^,)]*)\);', + r'(?:NimStringDesc \*|NimStringV2)\s?(dollar__[A-z0-9_]+?)\(([^,)]*)\);', gdb.execute("info functions dollar__", True, True) ) @@ -135,25 +160,23 @@ class DollarPrintFunction (gdb.Function): @staticmethod - def invoke_static(arg): - - if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name == "NimStringDesc": + def invoke_static(arg, ignore_errors = False): + if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg - argTypeName = str(arg.type) - for func, arg_typ in DollarPrintFunction.dollar_functions: # this way of overload resolution cannot deal with type aliases, # therefore it won't find all overloads. if arg_typ == argTypeName: - func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() + func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTION_DOMAIN).value() return func_value(arg) elif arg_typ == argTypeName + " *": - func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() + func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTION_DOMAIN).value() return func_value(arg.address) - printErrorOnce(argTypeName, "No suitable Nim $ operator found for type: " + argTypeName + "\n") + if not ignore_errors: + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -174,11 +197,11 @@ class NimStringEqFunction (gdb.Function): @staticmethod def invoke_static(arg1,arg2): - if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name == "NimStringDesc": + if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name in NIM_STRING_TYPES: str1 = NimStringPrinter(arg1).to_string() else: str1 = arg1.string() - if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name == "NimStringDesc": + if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name in NIM_STRING_TYPES: str2 = NimStringPrinter(arg1).to_string() else: str2 = arg2.string() @@ -190,6 +213,7 @@ class NimStringEqFunction (gdb.Function): NimStringEqFunction() + ################################################################################ ##### GDB Command, equivalent of Nim's $ operator ################################################################################ @@ -205,7 +229,7 @@ class DollarPrintCmd (gdb.Command): strValue = DollarPrintFunction.invoke_static(param) if strValue: gdb.write( - NimStringPrinter(strValue).to_string() + "\n", + str(NimStringPrinter(strValue)) + "\n", gdb.STDOUT ) @@ -243,7 +267,6 @@ class KochCmd (gdb.Command): os.path.dirname(os.path.dirname(__file__)), "koch") def invoke(self, argument, from_tty): - import os subprocess.run([self.binary] + gdb.string_to_argv(argument)) KochCmd() @@ -297,8 +320,14 @@ class NimBoolPrinter: ################################################################################ +def strFromLazy(strVal): + if isinstance(strVal, str): + return strVal + else: + return strVal.value().string("utf-8") + class NimStringPrinter: - pattern = re.compile(r'^NimStringDesc \*$') + pattern = re.compile(r'^(NimStringDesc \*|NimStringV2)$') def __init__(self, val): self.val = val @@ -308,10 +337,18 @@ class NimStringPrinter: def to_string(self): if self.val: - l = int(self.val['Sup']['len']) - return self.val['data'][0].address.string("utf-8", "ignore", l) + if self.val.type.name == "NimStringV2": + l = int(self.val["len"]) + data = self.val["p"]["data"] + else: + l = int(self.val['Sup']['len']) + data = self.val["data"] + return data.lazy_string(encoding="utf-8", length=l) else: - return "" + return "" + + def __str__(self): + return strFromLazy(self.to_string()) class NimRopePrinter: pattern = re.compile(r'^tyObject_RopeObj__([A-Za-z0-9]*) \*$') @@ -334,58 +371,43 @@ class NimRopePrinter: ################################################################################ -# proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = -# ## Return string representation for enumeration values -# var n = typ.node -# if ntfEnumHole notin typ.flags: -# let o = e - n.sons[0].offset -# if o >= 0 and o <% typ.node.len: -# return $n.sons[o].name -# else: -# # ugh we need a slow linear search: -# var s = n.sons -# for i in 0 .. n.len-1: -# if s[i].offset == e: -# return $s[i].name -# result = $e & " (invalid data!)" - def reprEnum(e, typ): - """ this is a port of the nim runtime function `reprEnum` to python """ + # Casts the value to the enum type and then calls the enum printer e = int(e) - n = typ["node"] - flags = int(typ["flags"]) - # 1 << 2 is {ntfEnumHole} - if ((1 << 2) & flags) == 0: - o = e - int(n["sons"][0]["offset"]) - if o >= 0 and 0 < int(n["len"]): - return n["sons"][o]["name"].string("utf-8", "ignore") - else: - # ugh we need a slow linear search: - s = n["sons"] - for i in range(0, int(n["len"])): - if int(s[i]["offset"]) == e: - return s[i]["name"].string("utf-8", "ignore") + val = gdb.Value(e).cast(typ) + return strFromLazy(NimEnumPrinter(val).to_string()) - return str(e) + " (invalid data!)" +def enumNti(typeNimName, idString): + typeInfoName = "NTI" + typeNimName.lower() + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + if nti is None: + typeInfoName = "NTI" + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + return (typeInfoName, nti) class NimEnumPrinter: - pattern = re.compile(r'^tyEnum_(\w*)__([A-Za-z0-9]*)$') + pattern = re.compile(r'^tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') + enumReprProc = gdb.lookup_global_symbol("reprEnum", gdb.SYMBOL_FUNCTION_DOMAIN) def __init__(self, val): - self.val = val - match = self.pattern.match(self.val.type.name) + self.val = val + typeName = self.val.type.name + match = self.pattern.match(typeName) self.typeNimName = match.group(1) - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) - - if self.nti is None: - printErrorOnce(typeInfoName, "NimEnumPrinter: lookup global symbol '" + typeInfoName + " failed for " + self.val.type.name + ".\n") + typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) def to_string(self): - if self.nti: - arg0 = self.val - arg1 = self.nti.value(gdb.newest_frame()) - return reprEnum(arg0, arg1) + if NimEnumPrinter.enumReprProc and self.nti: + # Use the old runtimes enumRepr function. + # We call the Nim proc itself so that the implementation is correct + f = gdb.newest_frame() + # We need to strip the quotes so it looks like an enum instead of a string + reprProc = NimEnumPrinter.enumReprProc.value() + return str(reprProc(self.val, self.nti.value(f).address)).strip('"') + elif dollarResult := DollarPrintFunction.invoke_static(self.val): + # New runtime doesn't use enumRepr so we instead try and call the + # dollar function for it + return str(NimStringPrinter(dollarResult)) else: return self.typeNimName + "(" + str(int(self.val)) + ")" @@ -395,34 +417,27 @@ class NimSetPrinter: ## the set printer is limited to sets that fit in an integer. Other ## sets are compiled to `NU8 *` (ptr uint8) and are invisible to ## gdb (currently). - pattern = re.compile(r'^tySet_tyEnum_(\w*)_([A-Za-z0-9]*)$') + pattern = re.compile(r'^tySet_tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') def __init__(self, val): self.val = val - match = self.pattern.match(self.val.type.name) - self.typeNimName = match.group(1) - - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) - - if self.nti is None: - printErrorOnce(typeInfoName, "NimSetPrinter: lookup global symbol '"+ typeInfoName +" failed for " + self.val.type.name + ".\n") + typeName = self.val.type.name + match = self.pattern.match(typeName) + self.typeNimName = match.group(1) def to_string(self): - if self.nti: - nti = self.nti.value(gdb.newest_frame()) - enumStrings = [] - val = int(self.val) - i = 0 - while val > 0: - if (val & 1) == 1: - enumStrings.append(reprEnum(i, nti)) - val = val >> 1 - i += 1 - - return '{' + ', '.join(enumStrings) + '}' - else: - return str(int(self.val)) + # Remove the tySet from the type name + typ = gdb.lookup_type(self.val.type.name[6:]) + enumStrings = [] + val = int(self.val) + i = 0 + while val > 0: + if (val & 1) == 1: + enumStrings.append(reprEnum(i, typ)) + val = val >> 1 + i += 1 + + return '{' + ', '.join(enumStrings) + '}' ################################################################################ @@ -454,33 +469,93 @@ class NimHashSetPrinter: ################################################################################ -class NimSeqPrinter: - # the pointer is explicity part of the type. So it is part of - # ``pattern``. - pattern = re.compile(r'^tySequence_\w* \*$') +class NimSeq: + # Wrapper around sequences. + # This handles the differences between old and new runtime def __init__(self, val): self.val = val + # new runtime has sequences on stack, old has them on heap + self.new = val.type.code != gdb.TYPE_CODE_PTR + if self.new: + # Some seqs are just the content and to save repeating ourselves we do + # handle them here. Only thing that needs to check this is the len/data getters + self.isContent = val.type.name.endswith("Content") + + def __bool__(self): + if self.new: + return self.val is not None + else: + return bool(self.val) + + def __len__(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + else: + return int(self.val["len"]) + else: + return self.val["Sup"]["len"] + + @property + def data(self): + if self.new: + if self.isContent: + return self.val["data"] + elif self.val["p"]: + return self.val["p"]["data"] + else: + return self.val["data"] + + @property + def cap(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + elif self.val["p"]: + return int(self.val["p"]["cap"]) + else: + return 0 + return int(self.val['Sup']['reserved']) + +class NimSeqPrinter: + pattern = re.compile(r'^tySequence_\w*\s?\*?$') + + def __init__(self, val): + self.val = NimSeq(val) + def display_hint(self): return 'array' def to_string(self): - len = 0 - cap = 0 - if self.val: - len = int(self.val['Sup']['len']) - cap = int(self.val['Sup']['reserved']) - - return 'seq({0}, {1})'.format(len, cap) + return f'seq({len(self.val)}, {self.val.cap})' def children(self): if self.val: - length = int(self.val['Sup']['len']) - #align = len(str(length - 1)) - for i in range(length): - yield ("data[{0}]".format(i), self.val["data"][i]) + val = self.val + length = len(val) + + if length <= 0: + return + + data = val.data + inaccessible = False + for i in range(length): + if inaccessible: + return + try: + str(data[i]) + yield "data[{0}]".format(i), data[i] + except RuntimeError: + inaccessible = True + yield "data[{0}]".format(i), "inaccessible" + ################################################################################ class NimArrayPrinter: @@ -524,9 +599,9 @@ class NimStringTablePrinter: def children(self): if self.val: - data = NimSeqPrinter(self.val['data']) + data = NimSeqPrinter(self.val['data'].referenced_value()) for idxStr, entry in data.children(): - if int(entry['Field2']) > 0: + if int(entry['Field0']) != 0: yield (idxStr + ".Field0", entry['Field0']) yield (idxStr + ".Field1", entry['Field1']) @@ -537,7 +612,6 @@ class NimTablePrinter: def __init__(self, val): self.val = val - # match = self.pattern.match(self.val.type.name) def display_hint(self): return 'map' @@ -548,7 +622,7 @@ class NimTablePrinter: if self.val: counter = int(self.val['counter']) if self.val['data']: - capacity = int(self.val['data']['Sup']['len']) + capacity = NimSeq(self.val["data"]).cap return 'Table({0}, {1})'.format(counter, capacity) @@ -556,66 +630,22 @@ class NimTablePrinter: if self.val: data = NimSeqPrinter(self.val['data']) for idxStr, entry in data.children(): - if int(entry['Field0']) > 0: + if int(entry['Field0']) != 0: yield (idxStr + '.Field1', entry['Field1']) yield (idxStr + '.Field2', entry['Field2']) +################################################################################ -################################################################ - -# this is untested, therefore disabled - -# class NimObjectPrinter: -# pattern = re.compile(r'^tyObject_.*$') - -# def __init__(self, val): -# self.val = val - -# def display_hint(self): -# return 'object' - -# def to_string(self): -# return str(self.val.type) - -# def children(self): -# if not self.val: -# yield "object", "<nil>" -# raise StopIteration - -# for (i, field) in enumerate(self.val.type.fields()): -# if field.type.code == gdb.TYPE_CODE_UNION: -# yield _union_field -# else: -# yield (field.name, self.val[field]) - -# def _union_field(self, i, field): -# rti = getNimRti(self.val.type.name) -# if rti is None: -# return (field.name, "UNION field can't be displayed without RTI") - -# node_sons = rti['node'].dereference()['sons'] -# prev_field = self.val.type.fields()[i - 1] - -# descriminant_node = None -# for i in range(int(node['len'])): -# son = node_sons[i].dereference() -# if son['name'].string("utf-8", "ignore") == str(prev_field.name): -# descriminant_node = son -# break -# if descriminant_node is None: -# raise ValueError("Can't find union descriminant field in object RTI") - -# if descriminant_node is None: raise ValueError("Can't find union field in object RTI") -# union_node = descriminant_node['sons'][int(self.val[prev_field])].dereference() -# union_val = self.val[field] - -# for f1 in union_val.type.fields(): -# for f2 in union_val[f1].type.fields(): -# if str(f2.name) == union_node['name'].string("utf-8", "ignore"): -# return (str(f2.name), union_val[f1][f2]) +class NimTuplePrinter: + pattern = re.compile(r"^tyTuple__([A-Za-z0-9]*)") -# raise ValueError("RTI is absent or incomplete, can't find union definition in RTI") + def __init__(self, val): + self.val = val + def to_string(self): + # We don't have field names so just print out the tuple as if it was anonymous + tupleValues = [str(self.val[field.name]) for field in self.val.type.fields()] + return f"({', '.join(tupleValues)})" ################################################################################ @@ -647,11 +677,11 @@ def makematcher(klass): return matcher def register_nim_pretty_printers_for_object(objfile): - nimMainSym = gdb.lookup_global_symbol("NimMain", gdb.SYMBOL_FUNCTIONS_DOMAIN) + nimMainSym = gdb.lookup_global_symbol("NimMain", gdb.SYMBOL_FUNCTION_DOMAIN) if nimMainSym and nimMainSym.symtab.objfile == objfile: print("set Nim pretty printers for ", objfile.filename) - objfile.type_printers = [NimTypePrinter()] + gdb.types.register_type_printer(objfile, NimTypePrinter()) objfile.pretty_printers = [makematcher(var) for var in list(globals().values()) if hasattr(var, 'pattern')] # Register pretty printers for all objfiles that are already loaded. diff --git a/tools/debug/nimlldb.py b/tools/debug/nimlldb.py new file mode 100644 index 000000000..4bc4e771f --- /dev/null +++ b/tools/debug/nimlldb.py @@ -0,0 +1,1380 @@ +import lldb +from collections import OrderedDict +from typing import Union + + +def sbvaluegetitem(self: lldb.SBValue, name: Union[int, str]) -> lldb.SBValue: + if isinstance(name, str): + return self.GetChildMemberWithName(name) + else: + return self.GetChildAtIndex(name) + + +# Make this easier to work with +lldb.SBValue.__getitem__ = sbvaluegetitem + +NIM_IS_V2 = True + + +def get_nti(value: lldb.SBValue, nim_name=None): + name_split = value.type.name.split("_") + type_nim_name = nim_name or name_split[1] + id_string = name_split[-1].split(" ")[0] + + type_info_name = "NTI" + type_nim_name.lower() + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + type_info_name = "NTI" + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + print(f"NimEnumPrinter: lookup global symbol: '{type_info_name}' failed for {value.type.name}.\n") + return type_nim_name, nti + + +def enum_to_string(value: lldb.SBValue, int_val=None, nim_name=None): + tname = nim_name or value.type.name.split("_")[1] + + enum_val = value.signed + if int_val is not None: + enum_val = int_val + + default_val = f"{tname}.{str(enum_val)}" + + fn_syms = value.target.FindFunctions("reprEnum") + if not fn_syms.GetSize() > 0: + return default_val + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + if arg_types.GetSize() < 2: + return default_val + + arg1_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + arg2_type: lldb.SBType = arg_types.GetTypeAtIndex(1) + + ty_info_name, nti = get_nti(value, nim_name=tname) + + if not nti.IsValid(): + return default_val + + call = f"{fn.name}(({arg1_type.name}){enum_val}, ({arg2_type.name})" + str(nti.GetLoadAddress()) + ");" + + res = executeCommand(call) + + if res.error.fail: + return default_val + + return f"{tname}.{res.summary[1:-1]}" + + +def to_string(value: lldb.SBValue): + # For getting NimStringDesc * value + value = value.GetNonSyntheticValue() + + # Check if data pointer is Null + if value.type.is_pointer and value.unsigned == 0: + return None + + size = int(value["Sup"]["len"].unsigned) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + data = value["data"] + + # Check if first element is NULL + base_data_type = value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(value.target.FindFirstType("char").GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def to_stringV2(value: lldb.SBValue): + # For getting NimStringV2 value + value = value.GetNonSyntheticValue() + + data = value["p"]["data"] + + # Check if data pointer is Null + if value["p"].unsigned == 0: + return None + + size = int(value["len"].signed) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(base_data_type.GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def NimString(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + if NIM_IS_V2: + res = to_stringV2(value) + else: + res = to_string(value) + + if res is not None: + return f'"{res}"' + else: + return "nil" + + +def rope_helper(value: lldb.SBValue) -> str: + value = value.GetNonSyntheticValue() + if value.type.is_pointer and value.unsigned == 0: + return "" + + if value["length"].unsigned == 0: + return "" + + if NIM_IS_V2: + str_val = to_stringV2(value["data"]) + else: + str_val = to_string(value["data"]) + + if str_val is None: + str_val = "" + + return rope_helper(value["left"]) + str_val + rope_helper(value["right"]) + + +def Rope(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + rope_str = rope_helper(value) + + if len(rope_str) == 0: + rope_str = "nil" + else: + rope_str = f'"{rope_str}"' + + return f"Rope({rope_str})" + + +def NCSTRING(value: lldb.SBValue, internal_dict=None): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + ty = value.Dereference().type + val = value.target.CreateValueFromAddress( + value.name or "temp", lldb.SBAddress(value.unsigned, value.target), ty + ).AddressOf() + return val.summary + + +def ObjectV2(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + orig_value = value.GetNonSyntheticValue() + if orig_value.type.is_pointer and orig_value.unsigned == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + while orig_value.type.is_pointer: + orig_value = orig_value.Dereference() + + if "_" in orig_value.type.name: + obj_name = orig_value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = orig_value.type.name + + num_children = value.num_children + fields = [] + + for i in range(num_children): + fields.append(f"{value[i].name}: {value[i].summary}") + + res = f"{obj_name}(" + ", ".join(fields) + ")" + return res + + +def Number(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + if value.type.is_pointer and value.signed == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.signed) + + +def Float(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def UnsignedNumber(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.unsigned) + + +def Bool(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def CharArray(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str([f"'{char}'" for char in value.uint8s]) + + +def Array(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + value = value.GetNonSyntheticValue() + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + value = value.GetNonSyntheticValue() + return "[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +def Tuple(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + while value.type.is_pointer: + value = value.Dereference() + + num_children = value.num_children + + fields = [] + + for i in range(num_children): + key = value[i].name + val = value[i].summary + if key.startswith("Field"): + fields.append(f"{val}") + else: + fields.append(f"{key}: {val}") + + return "(" + ", ".join(fields) + f")" + + +def is_local(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if line.file == decl.file and decl.line != 0: + return True + + return False + + +def is_in_scope(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if is_local(value) and decl.line < line.line: + return True + + return False + + +def Enum(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_value_summary(value) + if custom_summary is not None: + return custom_summary + + return enum_to_string(value) + + +def EnumSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.summary) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Set(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.value) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Table(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + return "Table({" + ", ".join(fields) + "})" + + +def HashSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + fields.append(f"{value[i].summary}") + + return "HashSet({" + ", ".join(fields) + "})" + + +def StringTable(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + fields = [] + + for i in range(value.num_children - 1): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + mode = value[value.num_children - 1].summary + + return "StringTable({" + ", ".join(fields) + f"}}, mode={mode})" + + +def Sequence(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return "@[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +class StringChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + if not NIM_IS_V2: + self.data_type = self.value.target.FindFirstType("char") + + self.first_element: lldb.SBValue + self.update() + self.count = 0 + + def num_children(self): + return self.count + + def get_child_index(self, name): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data_size + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> int: + if NIM_IS_V2: + if self.value["p"].unsigned == 0: + return 0 + + size = int(self.value["len"].signed) + + if size == 0: + return 0 + + data = self.value["p"]["data"] + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + else: + if self.value.type.is_pointer and self.value.unsigned == 0: + return 0 + + size = int(self.value["Sup"]["len"].unsigned) + + if size == 0: + return 0 + + data = self.value["data"] + + # Check if first element is NULL + base_data_type = self.value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + + return size + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + + data = self.get_data() + size = self.get_len() + + self.count = size + self.first_element = data + + if NIM_IS_V2: + self.data_type = data.type.GetArrayElementType().GetTypedefedType() + + self.data_size = self.data_type.GetByteSize() + + def has_children(self): + return bool(self.num_children()) + + +class ArrayChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + + def num_children(self): + return self.has_children() and self.value.num_children + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.value[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def update(self): + if not self.has_children(): + return + + self.first_element = self.value[0] + self.data_type = self.value.type.GetArrayElementType() + + def has_children(self): + if is_local(self.value): + if not is_in_scope(self.value): + return False + return bool(self.value.num_children) + + +class SeqChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.count = 0 + self.update() + + def num_children(self): + return self.count + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["len"] if NIM_IS_V2 else self.value["Sup"]["len"] + + def update(self): + self.count = 0 + + if is_local(self.value): + if not is_in_scope(self.value): + return + + self.count = self.get_len().unsigned + + if not self.has_children(): + return + + data = self.get_data() + self.data_type = data.type.GetArrayElementType() + + self.data = data.Cast(self.data_type.GetArrayType(self.num_children())) + self.first_element = self.data + + def has_children(self): + return bool(self.num_children()) + + +class ObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.children) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def populate_children(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + stack = [self.value.GetNonSyntheticValue()] + + index = 0 + + while stack: + cur_val = stack.pop() + if cur_val.type.is_pointer and cur_val.unsigned == 0: + continue + + while cur_val.type.is_pointer: + cur_val = cur_val.Dereference() + + # Add super objects if they exist + if cur_val.num_children > 0 and cur_val[0].name == "Sup" and cur_val[0].type.name.startswith("tyObject"): + stack.append(cur_val[0]) + + for child in cur_val.children: + child = child.GetNonSyntheticValue() + if child.name == "Sup": + continue + self.children[child.name] = index + self.child_list.append(child) + index += 1 + + def update(self): + self.populate_children() + + def has_children(self): + return bool(self.num_children()) + + +class HashSetChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + child = key.CreateValueFromAddress(f"[{str(index)}]", key.GetLoadAddress(), key.GetType()) + index += 1 + + self.child_list.append(child) + + def has_children(self): + return bool(self.num_children()) + + +class SetCharChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType("char") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + cur_pos = 0 + for child in self.value.children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data(f"[{len(self.child_list)}]", data, self.ty) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +def create_set_children(value: lldb.SBValue, child_type: lldb.SBType, starting_pos: int) -> list[lldb.SBValue]: + child_list: list[lldb.SBValue] = [] + cur_pos = starting_pos + + if value.num_children > 0: + children = value.children + else: + children = [value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = value.synthetic_child_from_data(f"[{len(child_list)}]", data, child_type) + child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + return child_list + + +class SetIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NI64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + bits = self.value.GetByteSize() * 8 + starting_pos = -(bits // 2) + self.child_list = create_set_children(self.value, self.ty, starting_pos) + + def has_children(self): + return bool(self.num_children()) + + +class SetUIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class SetEnumChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(self.value.type.name.replace("tySet_", "")) + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class TableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + val = el[2] + key_summary = key.summary + child = self.value.CreateValueFromAddress(key_summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key_summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class StringTableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[2].unsigned) + if field0 == 0: + continue + key = el[0] + val = el[1] + child = val.CreateValueFromAddress(key.summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + self.child_list.append(self.value["mode"]) + self.children["mode"] = index + + def has_children(self): + return bool(self.num_children()) + + +class LLDBDynamicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + value = value.GetNonSyntheticValue() + self.value: lldb.SBValue = value[0] + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + while self.value.type.is_pointer: + self.value = self.value.Dereference() + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.children.clear() + self.child_list = [] + + for i, child in enumerate(self.value.children): + name = child.name.strip('"') + new_child = child.CreateValueFromAddress(name, child.GetLoadAddress(), child.GetType()) + + self.children[name] = i + self.child_list.append(new_child) + + def has_children(self): + return bool(self.num_children()) + + +class LLDBBasicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = value + + def num_children(self): + if self.value is not None: + return self.value.num_children + return 0 + + def get_child_index(self, name: str): + return self.value.GetIndexOfChildWithName(name) + + def get_child_at_index(self, index): + return self.value.GetChildAtIndex(index) + + def update(self): + pass + + def has_children(self): + return self.num_children() > 0 + + +class CustomObjectChildrenProvider: + """ + This children provider handles values returned from lldbDebugSynthetic* + Nim procedures + """ + + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = get_custom_synthetic(value) or value + if "lldbdynamicobject" in self.value.type.name.lower(): + self.provider = LLDBDynamicObjectProvider(self.value, internalDict) + else: + self.provider = LLDBBasicObjectProvider(self.value, internalDict) + + def num_children(self): + return self.provider.num_children() + + def get_child_index(self, name: str): + return self.provider.get_child_index(name) + + def get_child_at_index(self, index): + return self.provider.get_child_at_index(index) + + def update(self): + self.provider.update() + + def has_children(self): + return self.provider.has_children() + + +def echo(debugger: lldb.SBDebugger, command: str, result, internal_dict): + debugger.HandleCommand("po " + command) + + +SUMMARY_FUNCTIONS: dict[str, lldb.SBFunction] = {} +SYNTHETIC_FUNCTIONS: dict[str, lldb.SBFunction] = {} + + +def get_custom_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SUMMARY_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + command = f"{fn.name}(({first_type.name})" + str(value.GetLoadAddress()) + ");" + else: + command = f"{fn.name}(*({first_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_value_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + + fn: lldb.SBFunction = SUMMARY_FUNCTIONS.get(value.type.name) + if fn is None: + return None + + command = f"{fn.name}(({value.type.name})" + str(value.signed) + ");" + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_synthetic(value: lldb.SBValue) -> Union[lldb.SBValue, None]: + """Get a custom synthetic object if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SYNTHETIC_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + first_arg = f"({first_type.name}){value.GetLoadAddress()}" + else: + first_arg = f"*({first_type.GetPointerType().name}){value.GetLoadAddress()}" + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + ret_type = arg_types.GetTypeAtIndex(1) + ret_type = get_base_type(ret_type) + + command = f""" + {ret_type.name} lldbT; + nimZeroMem((void*)(&lldbT), sizeof({ret_type.name})); + {fn.name}(({first_arg}), (&lldbT)); + lldbT; + """ + else: + command = f"{fn.name}({first_arg});" + + res = executeCommand(command) + + if res.error.fail: + print(res.error) + return None + + return res + + +def get_base_type(ty: lldb.SBType) -> lldb.SBType: + """Get the base type of the type""" + temp = ty + while temp.IsPointerType(): + temp = temp.GetPointeeType() + return temp + + +def use_base_type(ty: lldb.SBType) -> bool: + types_to_check = [ + "NF", + "NF32", + "NF64", + "NI", + "NI8", + "NI16", + "NI32", + "NI64", + "bool", + "NIM_BOOL", + "NU", + "NU8", + "NU16", + "NU32", + "NU64", + ] + + for type_to_check in types_to_check: + if ty.name.startswith(type_to_check): + return False + + return True + + +def breakpoint_function_wrapper(frame: lldb.SBFrame, bp_loc, internal_dict): + """This allows function calls to Nim for custom object summaries and synthetic children""" + debugger = lldb.debugger + + global SUMMARY_FUNCTIONS + global SYNTHETIC_FUNCTIONS + + global NIM_IS_V2 + + for tname, fn in SYNTHETIC_FUNCTIONS.items(): + debugger.HandleCommand(f"type synthetic delete -w nim {tname}") + + SUMMARY_FUNCTIONS = {} + SYNTHETIC_FUNCTIONS = {} + + target: lldb.SBTarget = debugger.GetSelectedTarget() + + NIM_IS_V2 = target.FindFirstType("TNimTypeV2").IsValid() + + module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + + for sym in module: + if ( + not sym.name.startswith("lldbDebugSummary") + and not sym.name.startswith("lldbDebugSynthetic") + and not sym.name.startswith("dollar___") + ): + continue + + fn_syms: lldb.SBSymbolContextList = target.FindFunctions(sym.name) + if not fn_syms.GetSize() > 0: + continue + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + pass # don't continue + elif arg_types.GetSize() != 1: + continue + + arg_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + if use_base_type(arg_type): + arg_type = get_base_type(arg_type) + + if sym.name.startswith("lldbDebugSummary") or sym.name.startswith("dollar___"): + SUMMARY_FUNCTIONS[arg_type.name] = fn + elif sym.name.startswith("lldbDebugSynthetic"): + SYNTHETIC_FUNCTIONS[arg_type.name] = fn + debugger.HandleCommand( + f"type synthetic add -w nim -l {__name__}.CustomObjectChildrenProvider {arg_type.name}" + ) + + +def executeCommand(command, *args): + debugger = lldb.debugger + process = debugger.GetSelectedTarget().GetProcess() + frame: lldb.SBFrame = process.GetSelectedThread().GetSelectedFrame() + + expr_options = lldb.SBExpressionOptions() + expr_options.SetIgnoreBreakpoints(False) + expr_options.SetFetchDynamicValue(lldb.eDynamicCanRunTarget) + expr_options.SetTimeoutInMicroSeconds(30 * 1000 * 1000) # 30 second timeout + expr_options.SetTryAllThreads(True) + expr_options.SetUnwindOnError(False) + expr_options.SetGenerateDebugInfo(True) + expr_options.SetLanguage(lldb.eLanguageTypeC) + expr_options.SetCoerceResultToId(True) + res = frame.EvaluateExpression(command, expr_options) + + return res + + +def __lldb_init_module(debugger, internal_dict): + # fmt: off + debugger.HandleCommand(f"breakpoint command add -F {__name__}.breakpoint_function_wrapper --script-type python 1") + debugger.HandleCommand(f"type summary add -w nim -n sequence -F {__name__}.Sequence -x tySequence_+[[:alnum:]]+$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SeqChildrenProvider -x tySequence_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n chararray -F {__name__}.CharArray -x char\s+[\d+]") + debugger.HandleCommand(f"type summary add -w nim -n array -F {__name__}.Array -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ArrayChildrenProvider -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n string -F {__name__}.NimString NimStringDesc") + + debugger.HandleCommand(f"type summary add -w nim -n stringv2 -F {__name__}.NimString -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringDesc$") + + debugger.HandleCommand(f"type summary add -w nim -n cstring -F {__name__}.NCSTRING NCSTRING") + + debugger.HandleCommand(f"type summary add -w nim -n object -F {__name__}.ObjectV2 -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ObjectChildrenProvider -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n tframe -F {__name__}.ObjectV2 -x TFrame$") + + debugger.HandleCommand(f"type summary add -w nim -n rootobj -F {__name__}.ObjectV2 -x RootObj$") + + debugger.HandleCommand(f"type summary add -w nim -n enum -F {__name__}.Enum -x tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n hashset -F {__name__}.HashSet -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.HashSetChildrenProvider -x tyObject_+HashSet_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n rope -F {__name__}.Rope -x tyObject_+Rope[[:alnum:]]+_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n setuint -F {__name__}.Set -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetIntChildrenProvider -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setint -F {__name__}.Set -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint2 -F {__name__}.Set -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setenum -F {__name__}.EnumSet -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetEnumChildrenProvider -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setchar -F {__name__}.Set -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetCharChildrenProvider -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n table -F {__name__}.Table -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.TableChildrenProvider -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n stringtable -F {__name__}.StringTable -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringTableChildrenProvider -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple2 -F {__name__}.Tuple -x tyObject_+Tuple_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple -F {__name__}.Tuple -x tyTuple_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n float -F {__name__}.Float NF") + debugger.HandleCommand(f"type summary add -w nim -n float32 -F {__name__}.Float NF32") + debugger.HandleCommand(f"type summary add -w nim -n float64 -F {__name__}.Float NF64") + debugger.HandleCommand(f"type summary add -w nim -n integer -F {__name__}.Number -x NI") + debugger.HandleCommand(f"type summary add -w nim -n integer8 -F {__name__}.Number -x NI8") + debugger.HandleCommand(f"type summary add -w nim -n integer16 -F {__name__}.Number -x NI16") + debugger.HandleCommand(f"type summary add -w nim -n integer32 -F {__name__}.Number -x NI32") + debugger.HandleCommand(f"type summary add -w nim -n integer64 -F {__name__}.Number -x NI64") + debugger.HandleCommand(f"type summary add -w nim -n bool -F {__name__}.Bool -x bool") + debugger.HandleCommand(f"type summary add -w nim -n bool2 -F {__name__}.Bool -x NIM_BOOL") + debugger.HandleCommand(f"type summary add -w nim -n uinteger -F {__name__}.UnsignedNumber -x NU") + debugger.HandleCommand(f"type summary add -w nim -n uinteger8 -F {__name__}.UnsignedNumber -x NU8") + debugger.HandleCommand(f"type summary add -w nim -n uinteger16 -F {__name__}.UnsignedNumber -x NU16") + debugger.HandleCommand(f"type summary add -w nim -n uinteger32 -F {__name__}.UnsignedNumber -x NU32") + debugger.HandleCommand(f"type summary add -w nim -n uinteger64 -F {__name__}.UnsignedNumber -x NU64") + debugger.HandleCommand("type category enable nim") + debugger.HandleCommand(f"command script add -f {__name__}.echo echo") + # fmt: on diff --git a/tools/deps.nim b/tools/deps.nim index 888237761..e43f7a2b4 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -1,22 +1,44 @@ -import os, uri, strformat +import std/[os, uri, strformat, strutils] +import std/private/gitutils + +when defined(nimPreviewSlimSystem): + import std/assertions proc exec(cmd: string) = echo "deps.cmd: " & cmd let status = execShellCmd(cmd) doAssert status == 0, cmd -const commitHead* = "HEAD" +proc execRetry(cmd: string) = + let ok = retryCall(call = block: + let status = execShellCmd(cmd) + let result = status == 0 + if not result: + echo fmt"failed command: '{cmd}', status: {status}" + result) + doAssert ok, cmd -proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, appendRepoName = true) = +proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, + appendRepoName = true, allowBundled = false) = let destDirBase = destDirBase.absolutePath let p = url.parseUri.path let name = p.splitFile.name var destDir = destDirBase if appendRepoName: destDir = destDir / name - let destDir2 = destDir.quoteShell + let quotedDestDir = destDir.quoteShell if not dirExists(destDir): # note: old code used `destDir / .git` but that wouldn't prevent git clone # from failing - exec fmt"git clone -q {url} {destDir2}" - exec fmt"git -C {destDir2} fetch -q" - exec fmt"git -C {destDir2} checkout -q {commit}" + execRetry fmt"git clone -q {url} {quotedDestDir}" + if isGitRepo(destDir): + let oldDir = getCurrentDir() + setCurrentDir(destDir) + try: + execRetry "git fetch -q" + exec fmt"git checkout -q {commit}" + finally: + setCurrentDir(oldDir) + elif allowBundled: + discard "this dependency was bundled with Nim, don't do anything" + else: + quit "FAILURE: " & destdir & " already exists but is not a git repo" diff --git a/tools/detect/detect.nim b/tools/detect/detect.nim index 841b3a675..ed9438494 100644 --- a/tools/detect/detect.nim +++ b/tools/detect/detect.nim @@ -12,7 +12,10 @@ # The second one is more portable, and less semantically correct. It only works # when there's a backing C compiler available as well, preventing standalone # compilation. -import os, strutils +import std/[os, strutils] + +when defined(nimPreviewSlimSystem): + import std/syncio when defined(openbsd) or defined(freebsd) or defined(netbsd): const @@ -615,6 +618,7 @@ v("MAP_POPULATE", no_other = true) header("<sys/resource.h>") v("RLIMIT_NOFILE") +v("RLIMIT_STACK") header("<sys/select.h>") v("FD_SETSIZE") @@ -626,6 +630,7 @@ v("MSG_EOR") v("MSG_OOB") v("SCM_RIGHTS") v("SO_ACCEPTCONN") +v("SO_BINDTODEVICE") v("SO_BROADCAST") v("SO_DEBUG") v("SO_DONTROUTE") diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4c4db4638..946945346 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,49 +1,52 @@ import dom import fuzzysearch +import std/[jsfetch, asyncjs] -proc textContent(e: Element): cstring {. - importcpp: "#.textContent", nodecl.} -proc textContent(e: Node): cstring {. - importcpp: "#.textContent", nodecl.} +proc setTheme(theme: cstring) {.exportc.} = + document.documentElement.setAttribute("data-theme", theme) + window.localStorage.setItem("theme", theme) -proc tree(tag: string; kids: varargs[Element]): Element = +# set `data-theme` attribute early to prevent white flash +setTheme: + let t = window.localStorage.getItem("theme") + if t.isNil: cstring"auto" else: t + +proc onDOMLoaded(e: Event) {.exportc.} = + # set theme select value + document.getElementById("theme-select").value = window.localStorage.getItem("theme") + + for pragmaDots in document.getElementsByClassName("pragmadots"): + pragmaDots.onclick = proc (event: Event) = + # Hide tease + event.target.parentNode.style.display = "none" + # Show actual + event.target.parentNode.nextSibling.style.display = "inline" + + +proc tree(tag: cstring; kids: varargs[Element]): Element = result = document.createElement tag for k in kids: result.appendChild k proc add(parent, kid: Element) = - if parent.nodeName == cstring"TR" and ( - kid.nodeName == cstring"TD" or kid.nodeName == cstring"TH"): + if parent.nodeName == "TR" and (kid.nodeName == "TD" or kid.nodeName == "TH"): let k = document.createElement("TD") appendChild(k, kid) appendChild(parent, k) else: appendChild(parent, kid) -proc setClass(e: Element; value: string) = +proc setClass(e: Element; value: cstring) = e.setAttribute("class", value) -proc text(s: string): Element = cast[Element](document.createTextNode(s)) proc text(s: cstring): Element = cast[Element](document.createTextNode(s)) -proc getElementById(id: cstring): Element {.importc: "document.getElementById", nodecl.} - proc replaceById(id: cstring; newTree: Node) = - let x = getElementById(id) + let x = document.getElementById(id) x.parentNode.replaceChild(newTree, x) newTree.id = id -proc findNodeWith(x: Element; tag, content: cstring): Element = - if x.nodeName == tag and x.textContent == content: - return x - for i in 0..<x.len: - let it = x[i] - let y = findNodeWith(it, tag, content) - if y != nil: return y - return nil - proc clone(e: Element): Element {.importcpp: "#.cloneNode(true)", nodecl.} -proc parent(e: Element): Element {.importcpp: "#.parentNode", nodecl.} proc markElement(x: Element) {.importcpp: "#.__karaxMarker__ = true", nodecl.} proc isMarked(x: Element): bool {. importcpp: "#.hasOwnProperty('__karaxMarker__')", nodecl.} @@ -52,20 +55,13 @@ proc title(x: Element): cstring {.importcpp: "#.title", nodecl.} proc sort[T](x: var openArray[T]; cmp: proc(a, b: T): int) {.importcpp: "#.sort(#)", nodecl.} -proc parentWith(x: Element; tag: cstring): Element = - result = x.parent - while result.nodeName != tag: - result = result.parent - if result == nil: return nil - proc extractItems(x: Element; items: var seq[Element]) = if x == nil: return - if x.nodeName == cstring"A": + if x.nodeName == "A": items.add x else: for i in 0..<x.len: - let it = x[i] - extractItems(it, items) + extractItems(x[i], items) # HTML trees are so shitty we transform the TOC into a decent # data-structure instead and work on that. @@ -76,16 +72,14 @@ type sortId: int doSort: bool -proc extractItems(x: TocEntry; heading: cstring; - items: var seq[Element]) = +proc extractItems(x: TocEntry; heading: cstring; items: var seq[Element]) = if x == nil: return if x.heading != nil and x.heading.textContent == heading: for i in 0..<x.kids.len: items.add x.kids[i].heading else: - for i in 0..<x.kids.len: - let it = x.kids[i] - extractItems(it, heading, items) + for k in x.kids: + extractItems(k, heading, items) proc toHtml(x: TocEntry; isRoot=false): Element = if x == nil: return nil @@ -119,31 +113,21 @@ proc toHtml(x: TocEntry; isRoot=false): Element = if ul.len != 0: result.add ul if result.len == 0: result = nil -#proc containsWord(a, b: cstring): bool {.asmNoStackFrame.} = - #{.emit: """ - #var escaped = `b`.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); - #return new RegExp("\\b" + escaped + "\\b").test(`a`); - #""".} - -proc isWhitespace(text: cstring): bool {.asmNoStackFrame.} = - {.emit: """ - return !/[^\s]/.test(`text`); - """.} +proc isWhitespace(text: cstring): bool {.importcpp: r"!/\S/.test(#)".} proc isWhitespace(x: Element): bool = - x.nodeName == cstring"#text" and x.textContent.isWhitespace or - x.nodeName == cstring"#comment" + x.nodeName == "#text" and x.textContent.isWhitespace or x.nodeName == "#comment" proc toToc(x: Element; father: TocEntry) = - if x.nodeName == cstring"UL": + if x.nodeName == "UL": let f = TocEntry(heading: nil, kids: @[], sortId: father.kids.len) var i = 0 while i < x.len: var nxt = i+1 while nxt < x.len and x[nxt].isWhitespace: inc nxt - if nxt < x.len and x[i].nodeName == cstring"LI" and x[i].len == 1 and - x[nxt].nodeName == cstring"UL": + if nxt < x.len and x[i].nodeName == "LI" and x[i].len == 1 and + x[nxt].nodeName == "UL": let e = TocEntry(heading: x[i][0], kids: @[], sortId: f.kids.len) let it = x[nxt] for j in 0..<it.len: @@ -156,13 +140,12 @@ proc toToc(x: Element; father: TocEntry) = father.kids.add f elif isWhitespace(x): discard - elif x.nodeName == cstring"LI": + elif x.nodeName == "LI": var idx: seq[int] = @[] for i in 0 ..< x.len: if not x[i].isWhitespace: idx.add i - if idx.len == 2 and x[idx[1]].nodeName == cstring"UL": - let e = TocEntry(heading: x[idx[0]], kids: @[], - sortId: father.kids.len) + if idx.len == 2 and x[idx[1]].nodeName == "UL": + let e = TocEntry(heading: x[idx[0]], kids: @[], sortId: father.kids.len) let it = x[idx[1]] for j in 0..<it.len: toToc(it[j], e) @@ -171,31 +154,25 @@ proc toToc(x: Element; father: TocEntry) = for i in 0..<x.len: toToc(x[i], father) else: - father.kids.add TocEntry(heading: x, kids: @[], - sortId: father.kids.len) + father.kids.add TocEntry(heading: x, kids: @[], sortId: father.kids.len) proc tocul(x: Element): Element = # x is a 'ul' element result = tree("UL") for i in 0..<x.len: let it = x[i] - if it.nodeName == cstring"LI": + if it.nodeName == "LI": result.add it.clone - elif it.nodeName == cstring"UL": + elif it.nodeName == "UL": result.add tocul(it) -proc getSection(toc: Element; name: cstring): Element = - let sec = findNodeWith(toc, "A", name) - if sec != nil: - result = sec.parentWith("LI") - proc uncovered(x: TocEntry): TocEntry = if x.kids.len == 0 and x.heading != nil: return if not isMarked(x.heading): x else: nil result = TocEntry(heading: x.heading, kids: @[], sortId: x.sortId, doSort: x.doSort) - for i in 0..<x.kids.len: - let y = uncovered(x.kids[i]) + for k in x.kids: + let y = uncovered(k) if y != nil: result.kids.add y if result.kids.len == 0: result = nil @@ -214,9 +191,8 @@ proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = t.markElement() for p in procs: if not isMarked(p): - let xx = getElementsByClass(p.parent, cstring"attachedType") + let xx = getElementsByClass(p.parentNode, "attachedType") if xx.len == 1 and xx[0].textContent == t.textContent: - #kout(cstring"found ", p.nodeName) let q = tree("A", text(p.title)) q.setAttr("href", p.getAttribute("href")) c.kids.add TocEntry(heading: q, kids: @[]) @@ -227,15 +203,13 @@ proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = var alternative: Element proc togglevis(d: Element) = - asm """ - if (`d`.style.display == 'none') - `d`.style.display = 'inline'; - else - `d`.style.display = 'none'; - """ + if d.style.display == "none": + d.style.display = "inline" + else: + d.style.display = "none" proc groupBy*(value: cstring) {.exportc.} = - let toc = getElementById("toc-list") + let toc = document.getElementById("toc-list") if alternative.isNil: var tt = TocEntry(heading: nil, kids: @[]) toToc(toc, tt) @@ -255,45 +229,38 @@ proc groupBy*(value: cstring) {.exportc.} = let ntoc = buildToc(tt, types, procs) let x = toHtml(ntoc, isRoot=true) alternative = tree("DIV", x) - if value == cstring"type": + if value == "type": replaceById("tocRoot", alternative) else: replaceById("tocRoot", tree("DIV")) - togglevis(getElementById"toc-list") + togglevis(document.getElementById"toc-list") var db: seq[Node] contents: seq[cstring] -template normalize(x: cstring): cstring = x.toLower.replace("_", "") + +proc escapeCString(x: var cstring) = + # Original strings are already escaped except HTML tags, so + # we only escape `<` and `>`. + var s = "" + for c in x: + case c + of '<': s.add("<") + of '>': s.add(">") + else: s.add(c) + x = s.cstring proc dosearch(value: cstring): Element = if db.len == 0: - var stuff: Element - {.emit: """ - var request = new XMLHttpRequest(); - request.open("GET", "theindex.html", false); - request.send(null); - - var doc = document.implementation.createHTMLDocument("theindex"); - doc.documentElement.innerHTML = request.responseText; - - //parser=new DOMParser(); - //doc=parser.parseFromString("<html></html>", "text/html"); - - `stuff` = doc.documentElement; - """.} - db = stuff.getElementsByClass"reference" - contents = @[] - for ahref in db: - contents.add ahref.getAttribute("data-doc-search-tag") + return let ul = tree("UL") result = tree("DIV") result.setClass"search_results" var matches: seq[(Node, int)] = @[] for i in 0..<db.len: let c = contents[i] - if c == cstring"Examples" or c == cstring"PEG construction": + if c == "Examples" or c == "PEG construction": # Some manual exclusions. # Ideally these should be fixed in the index to be more # descriptive of what they are. @@ -305,6 +272,7 @@ proc dosearch(value: cstring): Element = matches.sort(proc(a, b: auto): int = b[1] - a[1]) for i in 0 ..< min(matches.len, 29): matches[i][0].innerHTML = matches[i][0].getAttribute("data-doc-search-tag") + escapeCString(matches[i][0].innerHTML) ul.add(tree("LI", cast[Element](matches[i][0]))) if ul.len == 0: result.add tree("B", text"no search results") @@ -312,20 +280,103 @@ proc dosearch(value: cstring): Element = result.add tree("B", text"search results") result.add ul -var oldtoc: Element -var timer: Timeout +proc loadIndex() {.async.} = + ## Loads theindex.html to enable searching + let + indexURL = document.getElementById("indexLink").getAttribute("href") + # Get root of project documentation by cutting off theindex.html from index href + rootURL = ($indexURL)[0 ..< ^"theindex.html".len] + var resp = fetch(indexURL).await().text().await() + # Convert into element so we can use DOM functions to parse the html + var indexElem = document.createElement("div") + indexElem.innerHtml = resp + # Add items into the DB/contents + for href in indexElem.getElementsByClass("reference"): + # Make links be relative to project root instead of current page + href.setAttr("href", cstring(rootURL & $href.getAttribute("href"))) + db &= href + contents &= href.getAttribute("data-doc-search-tag") + + +var + oldtoc: Element + timer: Timeout + loadIndexFut: Future[void] = nil proc search*() {.exportc.} = proc wrapper() = - let elem = getElementById("searchInput") + let elem = document.getElementById("searchInput") let value = elem.value if value.len != 0: if oldtoc.isNil: - oldtoc = getElementById("tocRoot") + oldtoc = document.getElementById("tocRoot") let results = dosearch(value) replaceById("tocRoot", results) elif not oldtoc.isNil: replaceById("tocRoot", oldtoc) - + # Start loading index as soon as user starts typing. + # Will only be loaded the once anyways + if loadIndexFut == nil: + loadIndexFut = loadIndex() + # Run wrapper once loaded so we don't miss the users query + discard loadIndexFut.then(wrapper) if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) + +proc copyToClipboard*() {.exportc.} = + {.emit: """ + + function updatePreTags() { + + const allPreTags = document.querySelectorAll("pre:not(.line-nums)") + + allPreTags.forEach((e) => { + + const div = document.createElement("div") + div.classList.add("copyToClipBoard") + + const preTag = document.createElement("pre") + preTag.innerHTML = e.innerHTML + + const button = document.createElement("button") + button.value = e.textContent.replace('...', '') + button.classList.add("copyToClipBoardBtn") + button.style.cursor = "pointer" + + div.appendChild(preTag) + div.appendChild(button) + + e.outerHTML = div.outerHTML + + }) + } + + + function copyTextToClipboard(e) { + const clipBoardContent = e.target.value + navigator.clipboard.writeText(clipBoardContent).then(function() { + e.target.style.setProperty("--clipboard-image", "var(--clipboard-image-selected)") + }, function(err) { + console.error("Could not copy text: ", err); + }); + } + + window.addEventListener("click", (e) => { + if (e.target.classList.contains("copyToClipBoardBtn")) { + copyTextToClipboard(e) + } + }) + + window.addEventListener("mouseover", (e) => { + if (e.target.nodeName === "PRE") { + e.target.nextElementSibling.style.setProperty("--clipboard-image", "var(--clipboard-image-normal)") + } + }) + + window.addEventListener("DOMContentLoaded", updatePreTags) + + """ + .} + +copyToClipboard() +window.addEventListener("DOMContentLoaded", onDOMLoaded) diff --git a/tools/finish.nim b/tools/finish.nim index 4b5131045..69838829c 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -41,7 +41,7 @@ proc downloadMingw(): DownloadResult = let curl = findExe"curl" var cmd: string if curl.len > 0: - cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url + cmd = quoteShell(curl) & " --output " & "dist" / mingw & " " & url elif fileExists"bin/nimgrab.exe": cmd = r"bin\nimgrab.exe " & url & " dist" / mingw if cmd.len > 0: diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index d07c2bf8c..cbdc51efc 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -3,8 +3,11 @@ import std / [strutils, sets] +when defined(nimPreviewSlimSystem): + import std/syncio + import ".." / compiler / [ - llstream, ast, lexer, options, msgs, idents, + llstream, lexer, options, msgs, idents, lineinfos, pathutils] proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = @@ -13,12 +16,12 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = var stream = llStreamOpen(data) var declaredSyms = initHashSet[string]() var usedSyms = initHashSet[string]() + usedSyms.incl "module" # 'module' is the start rule. if stream != nil: declaredSyms.incl "section" # special case for 'section(RULE)' in the grammar var - L: TLexer - tok: TToken - initToken(tok) + L: Lexer + tok: Token openLexer(L, f, stream, cache, config) # load the first token: rawGetTok(L, tok) @@ -35,6 +38,10 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = usedSyms.incl word else: rawGetTok(L, tok) + for u in declaredSyms: + if u notin usedSyms: + echo "Unused non-terminal: ", u + for u in usedSyms: if u notin declaredSyms: echo "Undeclared non-terminal: ", u diff --git a/tools/heapdump2dot.nim b/tools/heapdump2dot.nim index 4cee6d674..6704cdfb0 100644 --- a/tools/heapdump2dot.nim +++ b/tools/heapdump2dot.nim @@ -1,5 +1,5 @@ -include prelude +include std/prelude proc main(input, output: string) = type NodeKind = enum diff --git a/tools/heapdumprepl.nim b/tools/heapdumprepl.nim index dbaef2f2c..4f06cf111 100644 --- a/tools/heapdumprepl.nim +++ b/tools/heapdumprepl.nim @@ -1,5 +1,4 @@ - -include prelude +include std/prelude import intsets type @@ -17,7 +16,6 @@ type roots: Table[int, NodeKind] proc add(father: Node; son: int) = - if father.kids.isNil: father.kids = @[] father.kids.add(son) proc renderNode(g: Graph; id: int) = @@ -141,8 +139,8 @@ proc importData(input: string): Graph = close(i) else: quit "error: cannot open " & input - shallowCopy(result.nodes, nodes) - shallowCopy(result.roots, roots) + result.nodes = move nodes + result.roots = move roots if paramCount() == 1: repl(importData(paramStr(1))) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 43b802ee2..477fb29fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,21 +1,33 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" const gaCode* = " --doc.googleAnalytics:UA-48159761-1" + paCode* = " --doc.plausibleAnalytics:nim-lang.org" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" var nimExe*: string +const allowList = ["jsbigints.nim", "jsheaders.nim", "jsformdata.nim", "jsfetch.nim", "jsutils.nim"] -template isJsOnly(file: string): bool = file.isRelativeTo("lib/js") +template isJsOnly(file: string): bool = + file.isRelativeTo("lib/js") or + file.extractFilename in allowList proc exe*(f: string): string = result = addFileExt(f, ExeExt) @@ -36,38 +48,20 @@ proc findNimImpl*(): tuple[path: string, ok: bool] = proc findNim*(): string = findNimImpl().path -proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPath - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - template inFold*(desc, body) = - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::group::" & desc elif existsEnv("TF_BUILD"): echo "##[group]" & desc - body - - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::endgroup::" elif existsEnv("TF_BUILD"): echo "##[endgroup]" proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = ## Execute shell command. Add log folding for various CI services. - # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 + let desc = if desc.len == 0: cmd else: desc inFold(desc): exec(cmd, errorcode, additionalPath) @@ -93,56 +87,53 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) -proc getRst2html(): seq[string] = +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md +""".splitWhitespace() + +proc getMd2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path - if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: - # maybe we should still show nimfix, could help reviving it + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: # `docs` is redundant with `overview`, might as well remove that file? result.add path - doAssert "doc/manual/var_t_return.rst".unixToNativePath in result # sanity check + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check const - pdf = """ -doc/manual.rst -doc/lib.rst -doc/tut1.rst -doc/tut2.rst -doc/tut3.rst -doc/nimc.rst -doc/niminst.rst -doc/gc.rst -""".splitWhitespace() - - doc0 = """ -lib/system/threads.nim -lib/system/channels.nim -""".splitWhitespace() # ran by `nim doc0` instead of `nim doc` + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md +""".splitWhitespace().mapIt("doc" / it) withoutIndex = """ -lib/wrappers/mysql.nim -lib/wrappers/iup.nim -lib/wrappers/sqlite3.nim -lib/wrappers/postgres.nim lib/wrappers/tinyc.nim -lib/wrappers/odbcsql.nim lib/wrappers/pcre.nim lib/wrappers/openssl.nim lib/posix/posix.nim lib/posix/linux.nim lib/posix/termios.nim -lib/js/jscore.nim """.splitWhitespace() # some of these are include files so shouldn't be docgen'd ignoredModules = """ -lib/prelude.nim lib/pure/future.nim lib/pure/collections/hashcommon.nim lib/pure/collections/tableimpl.nim @@ -158,8 +149,37 @@ lib/posix/posix_nintendoswitch_consts.nim lib/posix/posix_linux_amd64.nim lib/posix/posix_linux_amd64_consts.nim lib/posix/posix_other_consts.nim +lib/posix/posix_freertos_consts.nim lib/posix/posix_openbsd_amd64.nim lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim """.splitWhitespace() when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): @@ -170,23 +190,25 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): result = path.len > 0 and not ret.startsWith ".." proc getDocList(): seq[string] = + ## var docIgnore: HashSet[string] - for a in doc0: docIgnore.incl a for a in withoutIndex: docIgnore.incl a for a in ignoredModules: docIgnore.incl a # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim +lib/system/exceptions.nim lib/system/dollars.nim -lib/system/widestrs.nim +lib/system/ctypes.nim """.splitWhitespace() proc follow(a: PathEntry): bool = - a.path.lastPathPart notin ["nimcache", "htmldocs", "includes", "deprecated", "genode"] + result = a.path.lastPathPart notin ["nimcache", htmldocsDirname, + "includes", "deprecated", "genode"] and + not a.path.isRelativeTo("lib/fusion") # fusion was un-bundled but we need to keep this in case user has it installed for entry in walkDirRecFilter("lib", follow = follow): let a = entry.path if entry.kind != pcFile or a.splitFile.ext != ".nim" or @@ -220,105 +242,132 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: - let rst2html = getRst2html() + let rst2html = getMd2html() var - commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len) + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(doc0): - commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): - let extra = if isJsOnly(d): " --backend:js " else: "" + let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): - commands[i] = nim & " doc2 $# --git.url:$# -o:$# $#" % + commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % [nimArgs, gitUrl, destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). -proc buildPdfDoc*(nimArgs, destPath: string) = +proc nim2pdf(src: string, dst: string, nimArgs: string) = + # xxx expose as a `nim` command or in some other reusable way. + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + let texFile = outDir / src.lastPathPart.changeFileExt("tex") + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" + # `>` should work on windows, if not, we can use `execCmdEx` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) + moveFile(texFile.changeFileExt("pdf"), dst) + +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args + var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - echo "pdflatex not found; no PDF documentation generated" + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: - const pdflatexcmd = "pdflatex -interaction=nonstopmode " - for d in items(pdf): - exec(findNim().quoteShell() & " rst2tex $# $#" % [nimArgs, d]) - let tex = splitFile(d).name & ".tex" - removeFile("doc" / tex) - moveFile(tex, "doc" / tex) - # call LaTeX twice to get cross references right: - exec(pdflatexcmd & changeFileExt(d, "tex")) - exec(pdflatexcmd & changeFileExt(d, "tex")) - # delete all the crappy temporary files: - let pdf = splitFile(d).name & ".pdf" - let dest = destPath / pdf - removeFile(dest) - moveFile(dest=dest, source=pdf) - removeFile(changeFileExt(pdf, "aux")) - if fileExists(changeFileExt(pdf, "toc")): - removeFile(changeFileExt(pdf, "toc")) - removeFile(changeFileExt(pdf, "log")) - removeFile(changeFileExt(pdf, "out")) - removeFile(changeFileExt(d, "tex")) + for src in items(mdPdfList): + let dst = destPath / src.lastPathPart.changeFileExt("pdf") + pdfList.add dst + nim2pdf(src, dst, args) + echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose proc buildJS(): string = let nim = findNim() - exec(nim.quoteShell() & " js -d:release --out:$1 tools/nimblepkglist.nim" % - [webUploadOutput / "nimblepkglist.js"]) + exec("$# js -d:release --out:$# tools/nimblepkglist.nim" % + [nim.quoteShell(), webUploadOutput / "nimblepkglist.js"]) # xxx deadcode? and why is it only for webUploadOutput, not for local docs? result = getDocHacksJs(nimr = getCurrentDir(), nim) proc buildDocsDir*(args: string, dir: string) = let args = nimArgs & " " & args let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 38ed93f73..1c3670fd9 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -1,74 +1,120 @@ #compdef nim +# Installation note: +# Please name this file as _nim (with underscore!) and copy it to a +# completions directory, either: +# - system wide one, like /usr/share/zsh/functions/Completion/Unix/ on Linux +# - or to a user directory like ~/scripts. Then you also need to add +# that directory in your ~/.zshrc to `fpath` array like so: +# fpath=( ~/scripts "${fpath[@]}" ) + _nim() { - _arguments -C \ - ':command:(( - {compile,c}\:compile\ project\ with\ default\ code\ generator\ C - doc\:generate\ the\ documentation\ for\ inputfile - {compileToC,cc}\:compile\ project\ with\ C\ code\ generator - {compileToCpp,cpp}\:compile\ project\ to\ C++\ code - {compileToOC,objc}\:compile\ project\ to\ Objective\ C\ code - js\:compile\ project\ to\ Javascript - e\:run\ a\ Nimscript\ file - rst2html\:convert\ a\ reStructuredText\ file\ to\ HTML - rst2tex\:convert\ a\ reStructuredText\ file\ to\ TeX - jsondoc\:extract\ the\ documentation\ to\ a\ json\ file - buildIndex\:build\ an\ index\ for\ the\ whole\ documentation - genDepend\:generate\ a\ DOT\ file\ containing\ the\ module\ dependency\ graph - dump\:dump\ all\ defined\ conditionals\ and\ search\ paths - check\:checks\ the\ project\ for\ syntax\ and\ semantic - ))' \ - '*-r[run the application]' \ - '*--run[run the application]' \ - '*-p=[add path to search paths]' \ - '*--path=[add path to search paths]' \ - '*-d=[define a conditional symbol]' \ - '*--define=[define a conditional symbol]' \ - '*-u=[undefine a conditional symbol]' \ - '*--undef=[undefine a conditional symbol]' \ - '*-f[force rebuilding of all modules]' \ - '*--forceBuild[force rebuilding of all modules]' \ - '*--stackTrace=on[turn stack tracing on]' \ - '*--stackTrace=off[turn stack tracing off]' \ - '*--lineTrace=on[turn line tracing on]' \ - '*--lineTrace=off[turn line tracing off]' \ - '*--threads=on[turn support for multi-threading on]' \ - '*--threads=off[turn support for multi-threading off]' \ - '*-x=on[turn all runtime checks on]' \ - '*-x=off[turn all runtime checks off]' \ - '*--checks=on[turn all runtime checks on]' \ - '*--checks=off[turn all runtime checks off]' \ - '*--objChecks=on[turn obj conversion checks on]' \ - '*--objChecks=off[turn obj conversion checks off]' \ - '*--fieldChecks=on[turn case variant field checks on]' \ - '*--fieldChecks=off[turn case variant field checks off]' \ - '*--rangeChecks=on[turn range checks on]' \ - '*--rangeChecks=off[turn range checks off]' \ - '*--boundChecks=on[turn bound checks on]' \ - '*--boundChecks=off[turn bound checks off]' \ - '*--overflowChecks=on[turn int over-/underflow checks on]' \ - '*--overflowChecks=off[turn int over-/underflow checks off]' \ - '*-a[turn assertions on]' \ - '*-a[turn assertions off]' \ - '*--assertions=on[turn assertions on]' \ - '*--assertions=off[turn assertions off]' \ - '*--floatChecks=on[turn all floating point (NaN/Inf) checks on]' \ - '*--floatChecks=off[turn all floating point (NaN/Inf) checks off]' \ - '*--nanChecks=on[turn NaN checks on]' \ - '*--nanChecks=off[turn NaN checks off]' \ - '*--infChecks=on[turn Inf checks on]' \ - '*--infChecks=off[turn Inf checks off]' \ - '*--nilChecks=on[turn nil checks on]' \ - '*--nilChecks=off[turn nil checks off]' \ - '*--opt=none[do not optimize]' \ - '*--opt=speed[optimize for speed|size - use -d:release for a release build]' \ - '*--opt=size[optimize for size]' \ - '*--debugger:native[use native debugger (gdb)]' \ - '*--app=console[generate a console app]' \ - '*--app=gui[generate a GUI app]' \ - '*--app=lib[generate a dynamic library]' \ - '*--app=staticlib[generate a static library]' \ - ':filename:_files -g"*.nim"' + local -a nimCommands=( + {compile,c}:'compile project with default code generator C' + {compileToC,cc}:'compile project with C code generator' + {compileToCpp,cpp}:'compile project to C++ code' + {compileToOC,objc}:'compile project to Objective C code' + 'js:compile project to Javascript' + 'e:run a Nimscript file' + 'doc:generate the HTML documentation for inputfile' + 'rst2html:convert a reStructuredText file to HTML' + 'doc2tex:generate the documentation for inputfile to LaTeX' + 'rst2tex:convert a reStructuredText file to TeX' + 'jsondoc:extract the documentation to a json file' + 'buildIndex:build an index for the whole documentation' + 'genDepend:generate a DOT file containing the module dependency graph' + 'dump:dump all defined conditionals and search paths' + 'check:checks the project for syntax and semantic' + {--help,-h}:'basic help' + '--fullhelp:show all switches' + {-v,--version}:'show version' + ) + + _arguments '*:: :->anyState' && return 0 + + if (( CURRENT == 1 )); then + _describe -t commands "Nim commands" nimCommands -V1 + return + fi + + local -a conditionalSymbols=( + release danger mingw androidNDK useNimRtl useMalloc noSignalHandler ssl + debug leanCompiler gcDestructors) + local -a sharedOpts=( + {--define\\:-,-d\\:-}'[define a conditional symbol]:x:($conditionalSymbols)' + {--undef\\:-,-u\\:-}'[undefine a conditional symbol]:x:($conditionalSymbols)' + {--path\\:-,-p\\:-}'[add path to search paths]:x:_files' + '--verbosity\:-[set verbosity level (default\: 1)]:x:(0 1 2 3)' + '--hints\:-[print compilation hints? (or `list`)]:x:(on off list)' + ) + local runOpts=( + {--run,-r}'[run the application]' + ) + local docOpts=( + '--index\:-[enable index .idx files?]:x:(on off)' + '--project\:-[output any dependency for doc?]:x:(on off)' + '--docInternal\:-[generate module-private documentation?]:x:(on off)' + ) + local -a codeOpts=( + {--forceBuild,-f}'[force rebuilding of all modules]' + '--stackTrace\:-[enable stack tracing?]:x:(on off)' + '--lineTrace\:-[enable line tracing?]:x:(on off)' + '--threads\:-[enable support for multi-threading?]:x:(on off)' + {--checks\\:-,-x\\:-}'[enable/disable all runtime checks?]:x:(on off)' + '--objChecks\:-[enable obj conversion checks]:x:(on off)' + '--fieldChecks\:-[enable case variant field checks?]:x:(on off)' + '--rangeChecks\:-[enable range checks?]:x:(on off)' + '--boundChecks\:-[enable bound checks?]:x:(on off)' + '--overflowChecks\:-[enable integer over-/underflow checks?]:x:(on off)' + {--assertions\\:-,-a\\:-}'[enable assertions?]:x:(on off)' + '--floatChecks\:-[enable floating point (NaN/Inf) checks?]:x:(on off)' + '--nanChecks\:-[enable NaN checks?]:x:(on off)' + '--infChecks\:-[enable Inf checks?]:x:(on off)' + '--nilChecks\:-[enable nil checks?]:x:(on off)' + '--expandArc\:-[show how given proc looks before final backend pass]' + '--expandMacro\:-[dump every generated AST from given macro]' + ) + local -a nativeOpts=( + '--opt\:-[optimization mode]:x:(none speed size)' + '--debugger\:native[use native debugger (gdb)]' + '--app\:-[generate this type of app (lib=dynamic)]:x:(console gui lib staticlib)' + '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 vm wasm32)' + '--gc\:-[memory management algorithm to use (default\: refc)]:x:(refc arc orc markAndSweep boehm go regions none)' + '--os\:-[operating system to compile for]:x:(AIX Amiga Android Any Atari DOS DragonFly FreeBSD FreeRTOS Genode Haiku iOS Irix JS Linux MacOS MacOSX MorphOS NetBSD Netware NimVM NintendoSwitch OS2 OpenBSD PalmOS Standalone QNX SkyOS Solaris VxWorks Windows)' + '--panics\:-[turn panics into process termination (default\: off)]:x:(off on)' + ) + + case "$words[1]" in + compile|c|compileToC|cpp|compileToCpp|compileToOC|objc) + _arguments $codeOpts $runOpts $sharedOpts $nativeOpts \ + '*:filename:_files -g"*.nim"' + ;; + js) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nim"' + ;; + e) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nims"' + ;; + doc|doc2tex|jsondoc) + _arguments $runOpts $sharedOpts '*:filename:_files -g"*.nim"' + ;; + rst2html|rst2tex) + _arguments $runOpts $sharedOpts $docOpts '*:filename:_files -g"*.rst"' + ;; + buildIndex|genDepend|check) + _arguments $sharedOpts '*:filename:_files -g"*.nim"' + ;; + dump) + _arguments $sharedOpts + ;; + *) + _arguments '*:filename:_files -g"*"' + ;; + esac + + return 1 } _nim "$@" diff --git a/tools/nimblepkglist.nim b/tools/nimblepkglist.nim index c4bec4485..92e1cad20 100644 --- a/tools/nimblepkglist.nim +++ b/tools/nimblepkglist.nim @@ -56,16 +56,16 @@ proc processContent(content: string) = var officialPkgListDiv = document.getElementById("officialPkgList") officialPkgListDiv.innerHTML = - p("There are currently " & $officialCount & + (p("There are currently " & $officialCount & " official packages in the Nimble package repository.") & - ul(officialList) + ul(officialList)).cstring var unofficialPkgListDiv = document.getElementById("unofficialPkgList") unofficialPkgListDiv.innerHTML = - p("There are currently " & $unofficialCount & + (p("There are currently " & $unofficialCount & " unofficial packages in the Nimble package repository.") & - ul(unofficialList) + ul(unofficialList)).cstring proc gotPackageList(apiReply: TData) {.exportc.} = let decoded = decodeContent($apiReply.content) @@ -76,5 +76,5 @@ proc gotPackageList(apiReply: TData) {.exportc.} = var unofficialPkgListDiv = document.getElementById("unofficialPkgList") let msg = p("Unable to retrieve package list: ", code(getCurrentExceptionMsg())) - officialPkgListDiv.innerHTML = msg - unofficialPkgListDiv.innerHTML = msg + officialPkgListDiv.innerHTML = msg.cstring + unofficialPkgListDiv.innerHTML = msg.cstring diff --git a/tools/nimfind.nim b/tools/nimfind.nim deleted file mode 100644 index 05980d740..000000000 --- a/tools/nimfind.nim +++ /dev/null @@ -1,234 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2018 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nimfind is a tool that helps to give editors IDE like capabilities. - -when not defined(nimcore): - {.error: "nimcore MUST be defined for Nim's core tooling".} -when not defined(nimfind): - {.error: "nimfind MUST be defined for Nim's nimfind tool".} - -const Usage = """ -Nimfind - Tool to find declarations or usages for Nim symbols -Usage: - nimfind [options] file.nim:line:col - -Options: - --help, -h show this help - --rebuild rebuild the index - --project:file.nim use file.nim as the entry point - -In addition, all command line options of Nim that do not affect code generation -are supported. -""" - -import strutils, os, parseopt - -import "../compiler" / [options, commands, modules, sem, - passes, passaux, msgs, ast, - idents, modulegraphs, lineinfos, cmdlinehelper, - pathutils] - -import db_sqlite - -proc createDb(db: DbConn) = - db.exec(sql""" - create table if not exists filenames( - id integer primary key, - fullpath varchar(8000) not null - ); - """) - db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" - - # every sym can have potentially 2 different definitions due to forward - # declarations. - db.exec(sql""" - create table if not exists syms( - id integer primary key, - nimid integer not null, - name varchar(256) not null, - defline integer not null, - defcol integer not null, - deffile integer not null, - deflineB integer not null default 0, - defcolB integer not null default 0, - deffileB integer not null default 0, - foreign key (deffile) references filenames(id), - foreign key (deffileB) references filenames(id) - ); - """) - - db.exec(sql""" - create table if not exists usages( - id integer primary key, - nimid integer not null, - line integer not null, - col integer not null, - colB integer not null, - file integer not null, - foreign key (file) references filenames(id), - foreign key (nimid) references syms(nimid) - ); - """) - db.exec sql"create index if not exists UsagesIx on usages(file, line);" - -proc toDbFileId*(db: DbConn; conf: ConfigRef; fileIdx: FileIndex): int = - if fileIdx == FileIndex(-1): return -1 - let fullpath = toFullPath(conf, fileIdx) - let row = db.getRow(sql"select id from filenames where fullpath = ?", fullpath) - let id = row[0] - if id.len == 0: - result = int db.insertID(sql"insert into filenames(fullpath) values (?)", - fullpath) - else: - result = parseInt(id) - -type - FinderRef = ref object of RootObj - db: DbConn - -proc writeDef(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into syms(nimid, name, defline, defcol, deffile) values (?, ?, ?, ?, ?)""", - s.id, s.name.s, info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc writeDefResolveForward(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""update syms set deflineB = ?, defcolB = ?, deffileB = ? - where nimid = ?""", info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex), s.id) - -proc writeUsage(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into usages(nimid, line, col, colB, file) values (?, ?, ?, ?, ?)""", - s.id, info.line, info.col, info.col + s.name.s.len - 1, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc performSearch(conf: ConfigRef; dbfile: AbsoluteFile) = - var db = open(connection=string dbfile, user="nim", password="", - database="nim") - let pos = conf.m.trackPos - let fid = toDbFileId(db, conf, pos.fileIndex) - let known = toFullPath(conf, pos.fileIndex) - let nimids = db.getRow(sql"""select distinct nimid from usages where line = ? and file = ? and ? between col and colB""", - pos.line, fid, pos.col) - if nimids.len > 0: - var idSet = "" - for id in nimids: - if idSet.len > 0: idSet.add ", " - idSet.add id - var outputLater = "" - for r in db.rows(sql"""select line, col, filenames.fullpath from usages - inner join filenames on filenames.id = file - where nimid in (?)""", idSet): - let line = parseInt(r[0]) - let col = parseInt(r[1]) - let file = r[2] - if file == known and line == pos.line.int: - # output the line we already know last: - outputLater.add file & ":" & $line & ":" & $(col+1) & "\n" - else: - echo file, ":", line, ":", col+1 - if outputLater.len > 0: stdout.write outputLater - close(db) - -proc setupDb(g: ModuleGraph; dbfile: AbsoluteFile) = - var f = FinderRef() - removeFile(dbfile) - f.db = open(connection=string dbfile, user="nim", password="", - database="nim") - createDb(f.db) - f.db.exec(sql"pragma journal_mode=off") - # This MUST be turned off, otherwise it's way too slow even for testing purposes: - f.db.exec(sql"pragma SYNCHRONOUS=off") - f.db.exec(sql"pragma LOCKING_MODE=exclusive") - g.backend = f - -proc mainCommand(graph: ModuleGraph) = - let conf = graph.config - let dbfile = getNimcacheDir(conf) / RelativeFile"nimfind.db" - if not fileExists(dbfile) or optForceFullMake in conf.globalOptions: - clearPasses(graph) - registerPass graph, verbosePass - registerPass graph, semPass - conf.cmd = cmdIdeTools - wantMainModule(conf) - setupDb(graph, dbfile) - - graph.onDefinition = writeUsage # writeDef - graph.onDefinitionResolveForward = writeUsage # writeDefResolveForward - graph.onUsage = writeUsage - - if not fileExists(conf.projectFull): - quit "cannot find file: " & conf.projectFull.string - add(conf.searchPaths, conf.libpath) - conf.setErrorMaxHighMaybe - try: - compileProject(graph) - finally: - close(FinderRef(graph.backend).db) - performSearch(conf, dbfile) - -proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = - var p = parseopt.initOptParser(cmd) - while true: - parseopt.next(p) - case p.kind - of cmdEnd: break - of cmdLongOption, cmdShortOption: - case p.key.normalize - of "help", "h": - stdout.writeLine(Usage) - quit() - of "project": - conf.projectName = p.val - of "rebuild": - incl conf.globalOptions, optForceFullMake - else: processSwitch(pass, p, conf) - of cmdArgument: - let info = p.key.split(':') - if info.len == 3: - let (dir, file, ext) = info[0].splitFile() - conf.projectName = findProjectNimFile(conf, dir) - if conf.projectName.len == 0: conf.projectName = info[0] - try: - conf.m.trackPos = newLineInfo(conf, AbsoluteFile info[0], - parseInt(info[1]), parseInt(info[2])-1) - except ValueError: - quit "invalid command line" - else: - quit "invalid command line" - -proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = - let self = NimProg( - suggestMode: true, - processCmdLine: processCmdLine, - mainCommand: mainCommand - ) - self.initDefinesProg(conf, "nimfind") - - if paramCount() == 0: - stdout.writeLine(Usage) - return - - self.processCmdLineAndProjectPath(conf) - - # Find Nim's prefix dir. - let binaryPath = findExe("nim") - if binaryPath == "": - raise newException(IOError, - "Cannot find Nim standard library: Nim compiler not in PATH") - conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir() - if not dirExists(conf.prefixDir / RelativeDir"lib"): - conf.prefixDir = AbsoluteDir"" - - discard self.loadConfigsAndRunMainCommand(cache, conf) - -handleCmdLine(newIdentCache(), newConfigRef()) diff --git a/tools/nimfind.nims b/tools/nimfind.nims deleted file mode 100644 index 3013c360d..000000000 --- a/tools/nimfind.nims +++ /dev/null @@ -1,2 +0,0 @@ ---define:nimfind ---define:nimcore diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index ee5eced1e..c86159739 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -1,33 +1,21 @@ - - -when defined(windows): - import os, urldownloader - - proc syncDownload(url, file: string) = - proc progress(status: DownloadStatus, progress: uint, total: uint, - message: string) {.gcsafe.} = - echo "Downloading " & url - let t = total.BiggestInt - if t != 0: - echo clamp(int(progress.BiggestInt*100 div t), 0, 100), "%" - else: - echo "0%" - - downloadToFile(url, file, {optUseCache}, progress) - echo "100%" - -else: - import os, asyncdispatch, httpclient - - proc syncDownload(url, file: string) = - var client = newHttpClient() - proc onProgressChanged(total, progress, speed: BiggestInt) = - echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" - echo clamp(int(progress*100 div total), 0, 100), "%" - - client.onProgressChanged = onProgressChanged - client.downloadFile(url, file) - echo "100%" +import std/[os, httpclient] + +proc syncDownload(url, file: string) = + let client = newHttpClient() + proc onProgressChanged(total, progress, speed: BiggestInt) = + var message = "Downloading " + message.add url + message.add ' ' + message.addInt speed div 1000 + message.add "kb/s\n" + message.add $clamp(int(progress * 100 div total), 0, 100) + message.add '%' + echo message + + client.onProgressChanged = onProgressChanged + client.downloadFile(url, file) + client.close() + echo "100%" if os.paramCount() != 2: quit "Usage: nimgrab <url> <file>" diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index 138f1680c..599c616ba 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -8,86 +8,156 @@ # import - os, strutils, parseopt, pegs, re, terminal + os, strutils, parseopt, pegs, re, terminal, osproc, tables, algorithm, times const - Version = "1.5" - Usage = "nimgrep - Nim Grep Utility Version " & Version & """ - - (c) 2012 Andreas Rumpf -Usage: - nimgrep [options] [pattern] [replacement] (file/directory)* -Options: - --find, -f find the pattern (default) - --replace, -! replace the pattern - --peg pattern is a peg - --re pattern is a regular expression (default) - --rex, -x use the "extended" syntax for the regular expression - so that whitespace is not significant - --recursive, -r process directories recursively - --follow follow all symlinks when processing recursively - --confirm confirm each occurrence/replacement; there is a chance - to abort any time without touching the file - --stdin read pattern from stdin (to avoid the shell's confusing - quoting rules) - --word, -w the match should have word boundaries (buggy for pegs!) - --ignoreCase, -i be case insensitive - --ignoreStyle, -y be style insensitive - --ext:EX1|EX2|... only search the files with the given extension(s), - empty one ("--ext") means files with missing extension - --noExt:EX1|... exclude files having given extension(s), use empty one to - skip files with no extension (like some binary files are) - --includeFile:PAT include only files whose names match the given regex PAT - --excludeFile:PAT skip files whose names match the given regex pattern PAT - --excludeDir:PAT skip directories whose names match the given regex PAT - --nocolor output will be given without any colours - --color[:always] force color even if output is redirected - --colorTheme:THEME select color THEME from 'simple' (default), - 'bnw' (black and white) ,'ack', or 'gnu' (GNU grep) - --afterContext:N, - -a:N print N lines of trailing context after every match - --beforeContext:N, - -b:N print N lines of leading context before every match - --context:N, -c:N print N lines of leading context before every match and - N lines of trailing context after it - --group, -g group matches by file - --newLine, -l display every matching line starting from a new line - --verbose be verbose: list every processed file - --filenames find the pattern in the filenames, not in the contents - of the file - --help, -h shows this help - --version, -v shows the version -""" + Version = "2.0.0" + Usage = "nimgrep - Nim Grep Searching and Replacement Utility Version " & + Version & """ + + (c) 2012-2020 Andreas Rumpf +""" & slurp "../doc/nimgrep_cmdline.txt" + +# Limitations / ideas / TODO: +# * No unicode support with --cols +# * Consider making --onlyAscii default, since dumping binary data has +# stability and security repercussions +# * Mode - reads entire buffer by whole from stdin, which is bad for streaming. +# To implement line-by-line reading after adding option to turn off +# multiline matches +# * Add some form of file pre-processing, e.g. feed binary files to utility +# `strings` and then do the search inside these strings +# * Add --showCol option to also show column (of match), not just line; it +# makes it easier when jump to line+col in an editor or on terminal + + +# Search results for a file are modelled by these levels: +# FileResult -> Block -> Output/Chunk -> SubLine +# +# 1. SubLine is an entire line or its part. +# +# 2. Chunk, which is a sequence of SubLine, represents a match and its +# surrounding context. +# Output is a Chunk or one of auxiliary results like an openError. +# +# 3. Block, which is a sequence of Chunks, is not present as a separate type. +# It will just be separated from another Block by newline when there is +# more than 3 lines in it. +# Here is an example of a Block where only 1 match is found and +# 1 line before and 1 line after of context are required: +# +# ...a_line_before...................................... <<<SubLine(Chunk 1) +# +# .......pre....... ....new_match.... .......post...... +# ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^ +# SubLine (Chunk 1) SubLine (Chunk 1) SubLine (Chunk 2) +# +# ...a_line_after....................................... <<<SubLine(Chunk 2) +# +# 4. FileResult is printed as a sequence of Blocks. +# However FileResult is represented as seq[Output] in the program. type TOption = enum optFind, optReplace, optPeg, optRegex, optRecursive, optConfirm, optStdin, optWord, optIgnoreCase, optIgnoreStyle, optVerbose, optFilenames, - optRex, optFollow + optRex, optFollow, optCount, optLimitChars, optPipe TOptions = set[TOption] TConfirmEnum = enum ceAbort, ceYes, ceAll, ceNo, ceNone + Bin = enum + biOn, biOnly, biOff Pattern = Regex | Peg - -using pattern: Pattern + MatchInfo = tuple[first: int, last: int; + lineBeg: int, lineEnd: int, match: string] + outputKind = enum + openError, rejected, justCount, + blockFirstMatch, blockNextMatch, blockEnd, fileContents, outputFileName + Output = object + case kind: outputKind + of openError: msg: string # file/directory not found + of rejected: reason: string # when the file contents do not pass + of justCount: matches: int # the only output for option --count + of blockFirstMatch, blockNextMatch: # the normal case: match itself + pre: string + match: MatchInfo + of blockEnd: # block ending right after prev. match + blockEnding: string + firstLine: int + # == last lineN of last match + of fileContents: # yielded for --replace only + buffer: string + of outputFileName: # yielded for --filenames when no + name: string # PATTERN was provided + Trequest = (int, string) + FileResult = seq[Output] + Tresult = tuple[finished: bool, fileNo: int, + filename: string, fileResult: FileResult] + WalkOpt = tuple # used for walking directories/producing paths + extensions: seq[string] + notExtensions: seq[string] + filename: seq[string] + notFilename: seq[string] + dirPath: seq[string] + notDirPath: seq[string] + dirname : seq[string] + notDirname : seq[string] + WalkOptComp[Pat] = tuple # a compiled version of the previous + filename: seq[Pat] + notFilename: seq[Pat] + dirname : seq[Pat] + notDirname : seq[Pat] + dirPath: seq[Pat] + notDirPath: seq[Pat] + SearchOpt = tuple # used for searching inside a file + patternSet: bool # To distinguish uninitialized/empty 'pattern' + pattern: string # Main PATTERN + inFile: seq[string] # --inFile, --inf + notInFile: seq[string] # --notinFile, --ninf + inContext: seq[string] # --inContext, --inc + notInContext: seq[string] # --notinContext, --ninc + checkBin: Bin # --bin, --text + SearchOptComp[Pat] = tuple # a compiled version of the previous + pattern: Pat + inFile: seq[Pat] + notInFile: seq[Pat] + inContext: seq[Pat] + notInContext: seq[Pat] + SinglePattern[PAT] = tuple # compile single pattern for replacef + pattern: PAT + Column = tuple # current column info for the cropping (--limit) feature + terminal: int # column in terminal emulator + file: int # column in file (for correct Tab processing) + overflowMatches: int var - filenames: seq[string] = @[] - pattern = "" + paths: seq[string] = @[] replacement = "" - extensions: seq[string] = @[] + replacementSet = false + # to distinguish between uninitialized 'replacement' and empty one options: TOptions = {optRegex} - skipExtensions: seq[string] = @[] - excludeFile: seq[Regex] - includeFile: seq[Regex] - excludeDir: seq[Regex] + walkOpt {.threadvar.}: WalkOpt + searchOpt {.threadvar.}: SearchOpt + sortTime = false + sortTimeOrder = SortOrder.Ascending useWriteStyled = true - oneline = true + oneline = true # turned off by --group + expandTabs = true # Tabs are expanded in oneline mode linesBefore = 0 linesAfter = 0 linesContext = 0 - colorTheme = "simple" newLine = false + gVar = (matches: 0, errors: 0, reallyReplace: true) + # gVar - variables that can change during search/replace + nWorkers = 0 # run in single thread by default + searchRequestsChan: Channel[Trequest] + resultsChan: Channel[Tresult] + colorTheme: string = "simple" + limitCharUsr = high(int) # don't limit line width by default + termWidth = 80 + optOnlyAscii = false + +searchOpt.checkBin = biOn proc ask(msg: string): string = stdout.write(msg) @@ -170,6 +240,17 @@ proc printBlockFile(s: string) = of "ack": stdout.styledWrite(styleUnderscore, fgGreen, s) of "gnu": stdout.styledWrite(styleUnderscore, fgMagenta, s) +proc printBold(s: string) = + whenColors: + stdout.styledWrite(styleBright, s) + +proc printSpecial(s: string) = + whenColors: + case colorTheme + of "simple", "bnw": + stdout.styledWrite(if s == " ": styleReverse else: styleBright, s) + of "ack", "gnu": stdout.styledWrite(styleReverse, fgBlue, bgDefault, s) + proc printError(s: string) = whenColors: case colorTheme @@ -177,8 +258,6 @@ proc printError(s: string) = of "ack", "gnu": stdout.styledWriteLine(styleReverse, fgRed, bgDefault, s) stdout.flushFile() -const alignment = 6 - proc printLineN(s: string, isMatch: bool) = whenColors: case colorTheme @@ -201,11 +280,6 @@ proc printBlockLineN(s: string) = of "ack": stdout.styledWrite(styleUnderscore, fgYellow, s) of "gnu": stdout.styledWrite(styleUnderscore, fgGreen, s) -type - SearchInfo = tuple[buf: string, filename: string] - MatchInfo = tuple[first: int, last: int; - lineBeg: int, lineEnd: int, match: string] - proc writeColored(s: string) = whenColors: case colorTheme @@ -216,14 +290,23 @@ proc writeColored(s: string) = of "ack": stdout.styledWrite(styleReverse, fgYellow, bgDefault, s) of "gnu": stdout.styledWrite(fgRed, s) +proc printContents(s: string, isMatch: bool) = + if isMatch: + writeColored(s) + else: + stdout.write(s) + proc writeArrow(s: string) = whenColors: stdout.styledWrite(styleReverse, s) +const alignment = 6 # selected so that file contents start at 8, i.e. + # Tabs expand correctly without additional care + proc blockHeader(filename: string, line: int|string, replMode=false) = if replMode: writeArrow(" ->\n") - elif newLine: + elif newLine and optFilenames notin options and optPipe notin options: if oneline: printBlockFile(filename) printBlockLineN(":" & $line & ":") @@ -231,234 +314,437 @@ proc blockHeader(filename: string, line: int|string, replMode=false) = printBlockLineN($line.`$`.align(alignment) & ":") stdout.write("\n") -proc lineHeader(filename: string, line: int|string, isMatch: bool) = +proc newLn(curCol: var Column) = + stdout.write("\n") + curCol.file = 0 + curCol.terminal = 0 + +# We reserve 10+3 chars on the right in --cols mode (optLimitChars). +# If the current match touches this right margin, subLine before it will +# be cropped (even if space is enough for subLine after the match — we +# currently don't have a way to know it since we get it afterwards). +const matchPaddingFromRight = 10 +const ellipsis = "..." + +proc lineHeader(filename: string, line: int|string, isMatch: bool, + curCol: var Column) = let lineSym = if isMatch: $line & ":" else: $line & " " - if not newLine: + if not newLine and optFilenames notin options and optPipe notin options: if oneline: printFile(filename) printLineN(":" & lineSym, isMatch) + curcol.terminal += filename.len + 1 + lineSym.len else: printLineN(lineSym.align(alignment+1), isMatch) - stdout.write(" ") + curcol.terminal += lineSym.align(alignment+1).len + stdout.write(" "); curCol.terminal += 1 + curCol.terminal = curCol.terminal mod termWidth + if optLimitChars in options and + curCol.terminal > limitCharUsr - matchPaddingFromRight - ellipsis.len: + newLn(curCol) + +proc reserveChars(mi: MatchInfo): int = + if optLimitChars in options: + let patternChars = afterPattern(mi.match, 0) + 1 + result = patternChars + ellipsis.len + matchPaddingFromRight + else: + result = 0 + +# Our substitutions of non-printable symbol to ASCII character are similar to +# those of programm 'less'. +const lowestAscii = 0x20 # lowest ASCII Latin printable symbol (@) +const largestAscii = 0x7e +const by2ascii = 2 # number of ASCII chars to represent chars < lowestAscii +const by3ascii = 3 # number of ASCII chars to represent chars > largestAscii + +proc printExpanded(s: string, curCol: var Column, isMatch: bool, + limitChar: int) = + # Print taking into account tabs and optOnlyAscii (and also optLimitChar: + # the proc called from printCropped but we need to check column < limitChar + # also here, since exact cut points are known only after tab expansion). + # With optOnlyAscii non-ascii chars are highlighted even in matches. + # + # use buffer because: + # 1) we need to print non-ascii character inside matches while keeping the + # amount of color escape sequences minimal. + # 2) there is a report that fwrite buffering is slow on MacOS + # https://github.com/nim-lang/Nim/pull/15612#discussion_r510538326 + const bufSize = 8192 # typical for fwrite too + var buffer: string + const normal = 0 + const special = 1 + var lastAdded = normal + template dumpBuf() = + if lastAdded == normal: + printContents(buffer, isMatch) + else: + printSpecial(buffer) + template addBuf(i: int, s: char|string, size: int) = + if lastAdded != i or buffer.len + size > bufSize: + dumpBuf() + buffer.setlen(0) + buffer.add s + lastAdded = i + for c in s: + let charsAllowed = limitChar - curCol.terminal + if charsAllowed <= 0: + break + if lowestAscii <= int(c) and int(c) <= largestAscii: # ASCII latin + addBuf(normal, c, 1) + curCol.file += 1; curCol.terminal += 1 + elif (not optOnlyAscii) and c != '\t': # the same, print raw + addBuf(normal, c, 1) + curCol.file += 1; curCol.terminal += 1 + elif c == '\t': + let spaces = 8 - (curCol.file mod 8) + let spacesAllowed = min(spaces, charsAllowed) + curCol.file += spaces + curCol.terminal += spacesAllowed + if expandTabs: + if optOnlyAscii: # print a nice box for tab + addBuf(special, " ", 1) + addBuf(normal, " ".repeat(spacesAllowed-1), spacesAllowed-1) + else: + addBuf(normal, " ".repeat(spacesAllowed), spacesAllowed) + else: + addBuf(normal, '\t', 1) + else: # substitute characters that are not ACSII Latin + if int(c) < lowestAscii: + let substitute = char(int(c) + 0x40) # use common "control codes" + addBuf(special, "^" & substitute, by2ascii) + curCol.terminal += by2ascii + else: # int(c) > largestAscii + curCol.terminal += by3ascii + let substitute = '\'' & c.BiggestUInt.toHex(2) + addBuf(special, substitute, by3ascii) + curCol.file += 1 + if buffer.len > 0: + dumpBuf() + +template nextCharacter(c: char, file: var int, term: var int) = + if lowestAscii <= int(c) and int(c) <= largestAscii: # ASCII latin + file += 1 + term += 1 + elif (not optOnlyAscii) and c != '\t': # the same, print raw + file += 1 + term += 1 + elif c == '\t': + term += 8 - (file mod 8) + file += 8 - (file mod 8) + elif int(c) < lowestAscii: + file += 1 + term += by2ascii + else: # int(c) > largestAscii: + file += 1 + term += by3ascii + +proc calcTermLen(s: string, firstCol: int, chars: int, fromLeft: bool): int = + # calculate additional length added by Tabs expansion and substitutions + var col = firstCol + var first, last: int + if fromLeft: + first = max(0, s.len - chars) + last = s.len - 1 + else: + first = 0 + last = min(s.len - 1, chars - 1) + for c in s[first .. last]: + nextCharacter(c, col, result) + +proc printCropped(s: string, curCol: var Column, fromLeft: bool, + limitChar: int, isMatch = false) = + # print line `s`, may be cropped if option --cols was set + const eL = ellipsis.len + if optLimitChars notin options: + if not expandTabs and not optOnlyAscii: # for speed mostly + printContents(s, isMatch) + else: + printExpanded(s, curCol, isMatch, limitChar) + else: # limit columns, expand Tabs is also forced + var charsAllowed = limitChar - curCol.terminal + if fromLeft and charsAllowed < eL: + charsAllowed = eL + if (not fromLeft) and charsAllowed <= 0: + # already overflown and ellipsis shold be in place + return + let fullLenWithin = calcTermLen(s, curCol.file, charsAllowed, fromLeft) + # additional length from Tabs and special symbols + let addLen = fullLenWithin - min(s.len, charsAllowed) + # determine that the string is guaranteed to fit within `charsAllowed` + let fits = + if s.len > charsAllowed: + false + else: + if isMatch: fullLenWithin <= charsAllowed - eL + else: fullLenWithin <= charsAllowed + if fits: + printExpanded(s, curCol, isMatch, limitChar = high(int)) + else: + if fromLeft: + printBold ellipsis + curCol.terminal += eL + # find position `pos` where the right side of line will fit charsAllowed + var col = 0 + var term = 0 + var pos = min(s.len, max(0, s.len - (charsAllowed - eL))) + while pos <= s.len - 1: + let c = s[pos] + nextCharacter(c, col, term) + if term >= addLen: + break + inc pos + curCol.file = pos + # TODO don't expand tabs when cropped from the left - difficult, meaningless + printExpanded(s[pos .. s.len - 1], curCol, isMatch, + limitChar = high(int)) + else: + let last = max(-1, min(s.len - 1, charsAllowed - eL - 1)) + printExpanded(s[0 .. last], curCol, isMatch, limitChar-eL) + let numDots = limitChar - curCol.terminal + printBold ".".repeat(numDots) + curCol.terminal = limitChar -proc printMatch(fileName: string, mi: MatchInfo) = - let lines = mi.match.splitLines() - for i, l in lines: +proc printMatch(fileName: string, mi: MatchInfo, curCol: var Column) = + let sLines = mi.match.splitLines() + for i, l in sLines: if i > 0: - lineHeader(filename, mi.lineBeg + i, isMatch = true) - writeColored(l) - if i < lines.len - 1: - stdout.write("\n") + lineHeader(filename, mi.lineBeg + i, isMatch = true, curCol) + let charsAllowed = limitCharUsr - curCol.terminal + if charsAllowed > 0: + printCropped(l, curCol, fromLeft = false, limitCharUsr, isMatch = true) + else: + curCol.overflowMatches += 1 + if i < sLines.len - 1: + newLn(curCol) + +proc getSubLinesBefore(buf: string, curMi: MatchInfo): string = + let first = beforePattern(buf, curMi.first-1, linesBefore+1) + result = substr(buf, first, curMi.first-1) -proc printLinesBefore(si: SearchInfo, curMi: MatchInfo, nLines: int, - replMode=false) = +proc printSubLinesBefore(filename: string, beforeMatch: string, lineBeg: int, + curCol: var Column, reserveChars: int, + replMode=false) = # start block: print 'linesBefore' lines before current match `curMi` - let first = beforePattern(si.buf, curMi.first-1, nLines) - let lines = splitLines(substr(si.buf, first, curMi.first-1)) - let startLine = curMi.lineBeg - lines.len + 1 - blockHeader(si.filename, curMi.lineBeg, replMode=replMode) - for i, l in lines: - lineHeader(si.filename, startLine + i, isMatch = (i == lines.len - 1)) - stdout.write(l) - if i < lines.len - 1: - stdout.write("\n") + let sLines = splitLines(beforeMatch) + let startLine = lineBeg - sLines.len + 1 + blockHeader(filename, lineBeg, replMode=replMode) + for i, l in sLines: + let isLastLine = i == sLines.len - 1 + lineHeader(filename, startLine + i, isMatch = isLastLine, curCol) + let limit = if isLastLine: limitCharUsr - reserveChars else: limitCharUsr + l.printCropped(curCol, fromLeft = isLastLine, limitChar = limit) + if not isLastLine: + newLn(curCol) + +proc getSubLinesAfter(buf: string, mi: MatchInfo): string = + let last = afterPattern(buf, mi.last+1, 1+linesAfter) + let skipByte = # workaround posix: suppress extra line at the end of file + if (last == buf.len-1 and buf.len >= 2 and + buf[^1] == '\l' and buf[^2] != '\c'): 1 + else: 0 + result = substr(buf, mi.last+1, last - skipByte) + +proc printOverflow(filename: string, line: int, curCol: var Column) = + if curCol.overflowMatches > 0: + lineHeader(filename, line, isMatch = true, curCol) + printBold("(" & $curCol.overflowMatches & " matches skipped)") + newLn(curCol) + curCol.overflowMatches = 0 -proc printLinesAfter(si: SearchInfo, mi: MatchInfo, nLines: int) = +proc printSubLinesAfter(filename: string, afterMatch: string, matchLineEnd: int, + curCol: var Column) = # finish block: print 'linesAfter' lines after match `mi` - let s = si.buf - let last = afterPattern(s, mi.last+1, nLines) - let lines = splitLines(substr(s, mi.last+1, last)) - if lines.len == 0: # EOF - stdout.write("\n") + let sLines = splitLines(afterMatch) + if sLines.len == 0: # EOF + newLn(curCol) else: - stdout.write(lines[0]) # complete the line after match itself - stdout.write("\n") - let skipLine = # workaround posix line ending at the end of file - if last == s.len-1 and s.len >= 2 and s[^1] == '\l' and s[^2] != '\c': 1 - else: 0 - for i in 1 ..< lines.len - skipLine: - lineHeader(si.filename, mi.lineEnd + i, isMatch = false) - stdout.write(lines[i]) - stdout.write("\n") - if linesAfter + linesBefore >= 2 and not newLine: stdout.write("\n") + sLines[0].printCropped(curCol, fromLeft = false, limitCharUsr) + # complete the line after the match itself + newLn(curCol) + printOverflow(filename, matchLineEnd, curCol) + for i in 1 ..< sLines.len: + lineHeader(filename, matchLineEnd + i, isMatch = false, curCol) + sLines[i].printCropped(curCol, fromLeft = false, limitCharUsr) + newLn(curCol) -proc printBetweenMatches(si: SearchInfo, prevMi: MatchInfo, curMi: MatchInfo) = +proc getSubLinesBetween(buf: string, prevMi: MatchInfo, + curMi: MatchInfo): string = + buf.substr(prevMi.last+1, curMi.first-1) + +proc printBetweenMatches(filename: string, betweenMatches: string, + lastLineBeg: int, + curCol: var Column, reserveChars: int) = # continue block: print between `prevMi` and `curMi` - let lines = si.buf.substr(prevMi.last+1, curMi.first-1).splitLines() - stdout.write(lines[0]) # finish the line of previous Match - if lines.len > 1: - stdout.write("\n") - for i in 1 ..< lines.len: - lineHeader(si.filename, prevMi.lineEnd + i, - isMatch = (i == lines.len - 1)) - stdout.write(lines[i]) - if i < lines.len - 1: - stdout.write("\n") - -proc printContextBetween(si: SearchInfo, prevMi, curMi: MatchInfo) = - # print context after previous match prevMi and before current match curMi - let nLinesBetween = curMi.lineBeg - prevMi.lineEnd - if nLinesBetween <= linesAfter + linesBefore + 1: # print as 1 block - printBetweenMatches(si, prevMi, curMi) - else: # finalize previous block and then print next block - printLinesAfter(si, prevMi, 1+linesAfter) - printLinesBefore(si, curMi, linesBefore+1) - -proc printReplacement(si: SearchInfo, mi: MatchInfo, repl: string, - showRepl: bool, curPos: int, + let sLines = betweenMatches.splitLines() + sLines[0].printCropped(curCol, fromLeft = false, limitCharUsr) + # finish the line of previous Match + if sLines.len > 1: + newLn(curCol) + printOverflow(filename, lastLineBeg - sLines.len + 1, curCol) + for i in 1 ..< sLines.len: + let isLastLine = i == sLines.len - 1 + lineHeader(filename, lastLineBeg - sLines.len + i + 1, + isMatch = isLastLine, curCol) + let limit = if isLastLine: limitCharUsr - reserveChars else: limitCharUsr + sLines[i].printCropped(curCol, fromLeft = isLastLine, limitChar = limit) + if not isLastLine: + newLn(curCol) + +proc printReplacement(fileName: string, buf: string, mi: MatchInfo, + repl: string, showRepl: bool, curPos: int, newBuf: string, curLine: int) = - printLinesBefore(si, mi, linesBefore+1) - printMatch(si.fileName, mi) - printLinesAfter(si, mi, 1+linesAfter) + var curCol: Column + printSubLinesBefore(fileName, getSubLinesBefore(buf, mi), mi.lineBeg, + curCol, reserveChars(mi)) + printMatch(fileName, mi, curCol) + printSubLinesAfter(fileName, getSubLinesAfter(buf, mi), mi.lineEnd, curCol) stdout.flushFile() if showRepl: - let newSi: SearchInfo = (buf: newBuf, filename: si.filename) let miForNewBuf: MatchInfo = (first: newBuf.len, last: newBuf.len, lineBeg: curLine, lineEnd: curLine, match: "") - printLinesBefore(newSi, miForNewBuf, linesBefore+1, replMode=true) + printSubLinesBefore(fileName, getSubLinesBefore(newBuf, miForNewBuf), + miForNewBuf.lineBeg, curCol, reserveChars(miForNewBuf), + replMode=true) let replLines = countLineBreaks(repl, 0, repl.len-1) let miFixLines: MatchInfo = (first: mi.first, last: mi.last, lineBeg: curLine, lineEnd: curLine + replLines, match: repl) - printMatch(si.fileName, miFixLines) - printLinesAfter(si, miFixLines, 1+linesAfter) + printMatch(fileName, miFixLines, curCol) + printSubLinesAfter(fileName, getSubLinesAfter(buf, miFixLines), + miFixLines.lineEnd, curCol) + if linesAfter + linesBefore >= 2 and not newLine: stdout.write("\n") stdout.flushFile() -proc doReplace(si: SearchInfo, mi: MatchInfo, i: int, r: string; - newBuf: var string, curLine: var int, reallyReplace: var bool) = - newBuf.add(si.buf.substr(i, mi.first-1)) - inc(curLine, countLineBreaks(si.buf, i, mi.first-1)) +proc replace1match(filename: string, buf: string, mi: MatchInfo, i: int, + r: string; newBuf: var string, curLine: var int): bool = + newBuf.add(buf.substr(i, mi.first-1)) + inc(curLine, countLineBreaks(buf, i, mi.first-1)) if optConfirm in options: - printReplacement(si, mi, r, showRepl=true, i, newBuf, curLine) + printReplacement(filename, buf, mi, r, showRepl=true, i, newBuf, curLine) case confirm() of ceAbort: quit(0) - of ceYes: reallyReplace = true + of ceYes: gVar.reallyReplace = true of ceAll: - reallyReplace = true + gVar.reallyReplace = true options.excl(optConfirm) of ceNo: - reallyReplace = false + gVar.reallyReplace = false of ceNone: - reallyReplace = false + gVar.reallyReplace = false options.excl(optConfirm) - else: - printReplacement(si, mi, r, showRepl=reallyReplace, i, newBuf, curLine) - if reallyReplace: + elif optPipe notin options: + printReplacement(filename, buf, mi, r, showRepl=gVar.reallyReplace, i, + newBuf, curLine) + if gVar.reallyReplace: + result = true newBuf.add(r) inc(curLine, countLineBreaks(r, 0, r.len-1)) else: newBuf.add(mi.match) inc(curLine, countLineBreaks(mi.match, 0, mi.match.len-1)) -proc processFile(pattern; filename: string; counter: var int, errors: var int) = - var filenameShown = false - template beforeHighlight = - if not filenameShown and optVerbose notin options and not oneline: - printBlockFile(filename) - stdout.write("\n") - stdout.flushFile() - filenameShown = true +template updateCounters(output: Output) = + case output.kind + of blockFirstMatch, blockNextMatch: inc(gVar.matches) + of justCount: inc(gVar.matches, output.matches) + of openError: inc(gVar.errors) + of rejected, blockEnd, fileContents, outputFileName: discard - var buffer: string - if optFilenames in options: - buffer = filename - else: - try: - buffer = system.readFile(filename) - except IOError: - printError "Error: cannot open file: " & filename - inc(errors) - return - if optVerbose in options: - printFile(filename) - stdout.write("\n") - stdout.flushFile() - var result: string +proc printInfo(filename:string, output: Output) = + case output.kind + of openError: + printError("cannot open path '" & filename & "': " & output.msg) + of rejected: + if optVerbose in options: + echo "(rejected: ", output.reason, ")" + of justCount: + echo " (" & $output.matches & " matches)" + of blockFirstMatch, blockNextMatch, blockEnd, fileContents, outputFileName: + discard - if optReplace in options: - result = newStringOfCap(buffer.len) +proc printOutput(filename: string, output: Output, curCol: var Column) = + case output.kind + of openError, rejected, justCount: printInfo(filename, output) + of fileContents: discard # impossible + of outputFileName: + printCropped(output.name, curCol, fromLeft=false, limitCharUsr) + newLn(curCol) + of blockFirstMatch: + printSubLinesBefore(filename, output.pre, output.match.lineBeg, + curCol, reserveChars(output.match)) + printMatch(filename, output.match, curCol) + of blockNextMatch: + printBetweenMatches(filename, output.pre, output.match.lineBeg, + curCol, reserveChars(output.match)) + printMatch(filename, output.match, curCol) + of blockEnd: + printSubLinesAfter(filename, output.blockEnding, output.firstLine, curCol) + if linesAfter + linesBefore >= 2 and not newLine and + optFilenames notin options: stdout.write("\n") - var lineRepl = 1 - let si: SearchInfo = (buf: buffer, filename: filename) +iterator searchFile(pattern: Pattern; buffer: string): Output = var prevMi, curMi: MatchInfo - curMi.lineEnd = 1 + prevMi.lineEnd = 1 var i = 0 var matches: array[0..re.MaxSubpatterns-1, string] for j in 0..high(matches): matches[j] = "" - var reallyReplace = true - while i < buffer.len: + while true: let t = findBounds(buffer, pattern, matches, i) if t.first < 0 or t.last < t.first: - if optReplace notin options and prevMi.lineBeg != 0: # finalize last match - printLinesAfter(si, prevMi, 1+linesAfter) - stdout.flushFile() + if prevMi.lineBeg != 0: # finalize last match + yield Output(kind: blockEnd, + blockEnding: getSubLinesAfter(buffer, prevMi), + firstLine: prevMi.lineEnd) break - let lineBeg = curMi.lineEnd + countLineBreaks(buffer, i, t.first-1) + let lineBeg = prevMi.lineEnd + countLineBreaks(buffer, i, t.first-1) curMi = (first: t.first, last: t.last, lineBeg: lineBeg, lineEnd: lineBeg + countLineBreaks(buffer, t.first, t.last), match: buffer.substr(t.first, t.last)) - beforeHighlight() - inc counter - if optReplace notin options: - if prevMi.lineBeg == 0: # no previous match, so no previous block to finalize - printLinesBefore(si, curMi, linesBefore+1) - else: - printContextBetween(si, prevMi, curMi) - printMatch(si.fileName, curMi) - if t.last == buffer.len - 1: - stdout.write("\n") - stdout.flushFile() + if prevMi.lineBeg == 0: # no prev. match, so no prev. block to finalize + let pre = getSubLinesBefore(buffer, curMi) + prevMi = curMi + yield Output(kind: blockFirstMatch, pre: pre, match: move(curMi)) else: - let r = replace(curMi.match, pattern, replacement % matches) - doReplace(si, curMi, i, r, result, lineRepl, reallyReplace) - + let nLinesBetween = curMi.lineBeg - prevMi.lineEnd + if nLinesBetween <= linesAfter + linesBefore + 1: # print as 1 block + let pre = getSubLinesBetween(buffer, prevMi, curMi) + prevMi = curMi + yield Output(kind: blockNextMatch, pre: pre, match: move(curMi)) + else: # finalize previous block and then print next block + let after = getSubLinesAfter(buffer, prevMi) + yield Output(kind: blockEnd, blockEnding: after, + firstLine: prevMi.lineEnd) + let pre = getSubLinesBefore(buffer, curMi) + prevMi = curMi + yield Output(kind: blockFirstMatch, + pre: pre, + match: move(curMi)) i = t.last+1 - prevMi = curMi + when typeof(pattern) is Regex: + if buffer.len > MaxReBufSize: + yield Output(kind: openError, msg: "PCRE size limit is " & $MaxReBufSize) - if optReplace in options: - result.add(substr(buffer, i)) # finalize new buffer after last match - var f: File - if open(f, filename, fmWrite): - f.write(result) - f.close() - else: - quit "cannot open file for overwriting: " & filename - -proc hasRightFileName(path: string): bool = - let filename = path.lastPathPart - let ex = filename.splitFile.ext.substr(1) # skip leading '.' - if extensions.len != 0: - var matched = false - for x in items(extensions): - if os.cmpPaths(x, ex) == 0: - matched = true - break - if not matched: return false - for x in items(skipExtensions): - if os.cmpPaths(x, ex) == 0: return false - if includeFile.len != 0: - var matched = false - for x in items(includeFile): - if filename.match(x): - matched = true - break - if not matched: return false - for x in items(excludeFile): - if filename.match(x): return false - result = true +func detectBin(buffer: string): bool = + for i in 0 ..< min(1024, buffer.len): + if buffer[i] == '\0': + return true -proc hasRightDirectory(path: string): bool = - let dirname = path.lastPathPart - for x in items(excludeDir): - if dirname.match(x): return false - result = true +proc compilePeg(initPattern: string): Peg = + var pattern = initPattern + if optWord in options: + pattern = r"(^ / !\letter)(" & pattern & r") !\letter" + if optIgnoreStyle in options: + pattern = "\\y " & pattern + elif optIgnoreCase in options: + pattern = "\\i " & pattern + result = peg(pattern) proc styleInsensitive(s: string): string = template addx = @@ -494,28 +780,438 @@ proc styleInsensitive(s: string): string = addx() else: addx() -proc walker(pattern; dir: string; counter: var int, errors: var int) = - if dirExists(dir): - for kind, path in walkDir(dir): +proc compileRegex(initPattern: string): Regex = + var pattern = initPattern + var reflags = {reStudy} + if optIgnoreStyle in options: + pattern = styleInsensitive(pattern) + if optWord in options: + # see https://github.com/nim-lang/Nim/issues/13528#issuecomment-592786443 + pattern = r"(^|\W)(:?" & pattern & r")($|\W)" + if {optIgnoreCase, optIgnoreStyle} * options != {}: + reflags.incl reIgnoreCase + result = if optRex in options: rex(pattern, reflags) + else: re(pattern, reflags) + +template declareCompiledPatterns(compiledStruct: untyped, + StructType: untyped, + body: untyped) = + {.hint[XDeclaredButNotUsed]: off.} + if optRegex notin options: + var compiledStruct: StructType[Peg] + template compile1Pattern(p: string, pat: Peg) = + if p!="": pat = p.compilePeg() + proc compileArray(initPattern: seq[string]): seq[Peg] = + for pat in initPattern: + result.add pat.compilePeg() + body + else: + var compiledStruct: StructType[Regex] + template compile1Pattern(p: string, pat: Regex) = + if p!="": pat = p.compileRegex() + proc compileArray(initPattern: seq[string]): seq[Regex] = + for pat in initPattern: + result.add pat.compileRegex() + body + {.hint[XDeclaredButNotUsed]: on.} + +template ensureIncluded(includePat: seq[Pattern], str: string, + body: untyped) = + if includePat.len != 0: + var matched = false + for pat in includePat: + if str.contains(pat): + matched = true + break + if not matched: + body + +template ensureExcluded(excludePat: seq[Pattern], str: string, + body: untyped) = + {.warning[UnreachableCode]: off.} + for pat in excludePat: + if str.contains(pat, 0): + body + break + {.warning[UnreachableCode]: on.} + +func checkContext(context: string, searchOptC: SearchOptComp[Pattern]): bool = + ensureIncluded searchOptC.inContext, context: + return false + ensureExcluded searchOptC.notInContext, context: + return false + result = true + +iterator processFile(searchOptC: SearchOptComp[Pattern], filename: string, + yieldContents=false): Output = + var buffer: string + + var error = false + if optFilenames in options: + buffer = filename + elif optPipe in options: + buffer = stdin.readAll() + else: + try: + buffer = system.readFile(filename) + except IOError as e: + yield Output(kind: openError, msg: "readFile failed") + error = true + + if not error: + var reject = false + var reason: string + if searchOpt.checkBin in {biOff, biOnly}: + let isBin = detectBin(buffer) + if isBin and searchOpt.checkBin == biOff: + reject = true + reason = "binary file" + if (not isBin) and searchOpt.checkBin == biOnly: + reject = true + reason = "text file" + + if not reject: + ensureIncluded searchOptC.inFile, buffer: + reject = true + reason = "doesn't contain a requested match" + + if not reject: + ensureExcluded searchOptC.notInFile, buffer: + reject = true + reason = "contains a forbidden match" + + if reject: + yield Output(kind: rejected, reason: move(reason)) + elif optFilenames in options and searchOpt.pattern == "": + yield Output(kind: outputFileName, name: move(buffer)) + else: + var found = false + var cnt = 0 + let skipCheckContext = (searchOpt.notInContext.len == 0 and + searchOpt.inContext.len == 0) + if skipCheckContext: + for output in searchFile(searchOptC.pattern, buffer): + found = true + if optCount notin options: + yield output + else: + if output.kind in {blockFirstMatch, blockNextMatch}: + inc(cnt) + else: + var context: string + var outputAccumulator: seq[Output] + for outp in searchFile(searchOptC.pattern, buffer): + if outp.kind in {blockFirstMatch, blockNextMatch}: + outputAccumulator.add outp + context.add outp.pre + context.add outp.match.match + elif outp.kind == blockEnd: + outputAccumulator.add outp + context.add outp.blockEnding + # context has been formed, now check it: + if checkContext(context, searchOptC): + found = true + for output in outputAccumulator: + if optCount notin options: + yield output + else: + if output.kind in {blockFirstMatch, blockNextMatch}: + inc(cnt) + context = "" + outputAccumulator.setLen 0 + # end `if skipCheckContext`. + if optCount in options and cnt > 0: + yield Output(kind: justCount, matches: cnt) + if yieldContents and found and optCount notin options: + yield Output(kind: fileContents, buffer: move(buffer)) + +proc hasRightPath(path: string, walkOptC: WalkOptComp[Pattern]): bool = + if not ( + walkOpt.extensions.len > 0 or walkOpt.notExtensions.len > 0 or + walkOpt.filename.len > 0 or walkOpt.notFilename.len > 0 or + walkOpt.notDirPath.len > 0 or walkOpt.dirPath.len > 0): + return true + let filename = path.lastPathPart + let ex = filename.splitFile.ext.substr(1) # skip leading '.' + if walkOpt.extensions.len != 0: + var matched = false + for x in walkOpt.extensions: + if os.cmpPaths(x, ex) == 0: + matched = true + break + if not matched: return false + for x in walkOpt.notExtensions: + if os.cmpPaths(x, ex) == 0: return false + ensureIncluded walkOptC.filename, filename: + return false + ensureExcluded walkOptC.notFilename, filename: + return false + let parent = path.parentDir + ensureExcluded walkOptC.notDirPath, parent: + return false + ensureIncluded walkOptC.dirPath, parent: + return false + result = true + +proc isRightDirectory(path: string, walkOptC: WalkOptComp[Pattern]): bool = + ## --dirname can be only checked when the final path is known + ## so this proc is suitable for files only. + if walkOptC.dirname.len > 0: + var badDirname = false + var (nextParent, dirname) = splitPath(path) + # check that --dirname matches for one of directories in parent path: + while dirname != "": + badDirname = false + ensureIncluded walkOptC.dirname, dirname: + badDirname = true + if not badDirname: + break + (nextParent, dirname) = splitPath(nextParent) + if badDirname: # badDirname was set to true for all the dirs + return false + result = true + +proc descendToDirectory(path: string, walkOptC: WalkOptComp[Pattern]): bool = + ## --notdirname can be checked for directories immediately for optimization to + ## prevent descending into undesired directories. + if walkOptC.notDirname.len > 0: + let dirname = path.lastPathPart + ensureExcluded walkOptC.notDirname, dirname: + return false + result = true + +iterator walkDirBasic(dir: string, walkOptC: WalkOptComp[Pattern]): string + {.closure.} = + var dirStack = @[dir] # stack of directories + var timeFiles = newSeq[(times.Time, string)]() + while dirStack.len > 0: + let d = dirStack.pop() + let rightDirForFiles = d.isRightDirectory(walkOptC) + var files = newSeq[string]() + var dirs = newSeq[string]() + for kind, path in walkDir(d, skipSpecial = true): case kind of pcFile: - if path.hasRightFileName: - processFile(pattern, path, counter, errors) + if path.hasRightPath(walkOptC) and rightDirForFiles: + files.add(path) of pcLinkToFile: - if optFollow in options and path.hasRightFileName: - processFile(pattern, path, counter, errors) + if optFollow in options and path.hasRightPath(walkOptC) and + rightDirForFiles: + files.add(path) of pcDir: - if optRecursive in options and path.hasRightDirectory: - walker(pattern, path, counter, errors) + if optRecursive in options and path.descendToDirectory(walkOptC): + dirs.add path of pcLinkToDir: if optFollow in options and optRecursive in options and - path.hasRightDirectory: - walker(pattern, path, counter, errors) - elif fileExists(dir): - processFile(pattern, dir, counter, errors) + path.descendToDirectory(walkOptC): + dirs.add path + if sortTime: # sort by time - collect files before yielding + for file in files: + var time: Time + try: + time = getLastModificationTime(file) # can fail for broken symlink + except: + discard + timeFiles.add((time, file)) + else: # alphanumeric sort, yield immediately after sorting + files.sort() + for file in files: + yield file + dirs.sort(order = SortOrder.Descending) + for dir in dirs: + dirStack.add(dir) + if sortTime: + timeFiles.sort(sortTimeOrder) + for (_, file) in timeFiles: + yield file + +iterator walkRec(paths: seq[string]): tuple[error: string, filename: string] + {.closure.} = + declareCompiledPatterns(walkOptC, WalkOptComp): + walkOptC.notFilename.add walkOpt.notFilename.compileArray() + walkOptC.filename.add walkOpt.filename.compileArray() + walkOptC.dirname.add walkOpt.dirname.compileArray() + walkOptC.notDirname.add walkOpt.notDirname.compileArray() + walkOptC.dirPath.add walkOpt.dirPath.compileArray() + walkOptC.notDirPath.add walkOpt.notDirPath.compileArray() + for path in paths: + if dirExists(path): + for p in walkDirBasic(path, walkOptC): + yield ("", p) + else: + yield ( + if fileExists(path): ("", path) + else: ("Error: no such file or directory: ", path)) + +proc replaceMatches(pattern: Pattern; filename: string, buffer: string, + fileResult: FileResult) = + var newBuf = newStringOfCap(buffer.len) + + var changed = false + var lineRepl = 1 + var i = 0 + for output in fileResult: + if output.kind in {blockFirstMatch, blockNextMatch}: + let curMi = output.match + let r = replacef(curMi.match, pattern, replacement) + if replace1match(filename, buffer, curMi, i, r, newBuf, lineRepl): + changed = true + i = curMi.last + 1 + if changed and optPipe notin options: + newBuf.add(substr(buffer, i)) # finalize new buffer after last match + var f: File + if open(f, filename, fmWrite): + f.write(newBuf) + f.close() + else: + printError "cannot open file for overwriting: " & filename + inc(gVar.errors) + elif optPipe in options: # always print new buffer to stdout in pipe mode + newBuf.add(substr(buffer, i)) # finalize new buffer after last match + stdout.write(newBuf) + +template processFileResult(pattern: Pattern; filename: string, + fileResult: untyped) = + var filenameShown = false + template showFilename = + if not filenameShown: + printBlockFile(filename) + stdout.write("\n") + stdout.flushFile() + filenameShown = true + if optVerbose in options: + showFilename + if optReplace notin options: + var curCol: Column + var toFlush: bool + for output in fileResult: + updateCounters(output) + toFlush = true + if output.kind notin {rejected, openError, justCount} and not oneline: + showFilename + if output.kind == justCount and oneline: + printFile(filename & ":") + printOutput(filename, output, curCol) + if nWorkers == 0 and output.kind in {blockFirstMatch, blockNextMatch}: + stdout.flushFile() # flush immediately in single thread mode + if toFlush: stdout.flushFile() else: - printError "Error: no such file or directory: " & dir - inc(errors) + var buffer = "" + var matches: FileResult + for output in fileResult: + updateCounters(output) + case output.kind + of rejected, openError, justCount, outputFileName: + printInfo(filename, output) + of blockFirstMatch, blockNextMatch, blockEnd: + matches.add(output) + of fileContents: buffer = output.buffer + if matches.len > 0: + replaceMatches(pattern, filename, buffer, matches) + +proc run1Thread() = + declareCompiledPatterns(searchOptC, SearchOptComp): + compile1Pattern(searchOpt.pattern, searchOptC.pattern) + searchOptC.inFile.add searchOpt.inFile.compileArray() + searchOptC.notInFile.add searchOpt.notInFile.compileArray() + searchOptC.inContext.add searchOpt.inContext.compileArray() + searchOptC.notInContext.add searchOpt.notInContext.compileArray() + if optPipe in options: + processFileResult(searchOptC.pattern, "-", + processFile(searchOptC, "-", + yieldContents=optReplace in options)) + for entry in walkRec(paths): + if entry.error != "": + inc(gVar.errors) + printError (entry.error & entry.filename) + continue + processFileResult(searchOptC.pattern, entry.filename, + processFile(searchOptC, entry.filename, + yieldContents=optReplace in options)) + +# Multi-threaded version: all printing is being done in the Main thread. +# Totally nWorkers+1 additional threads are created (workers + pathProducer). +# An example of case nWorkers=2: +# +# ------------------ initial paths ------------------- +# | Main thread |----------------->| pathProducer | +# ------------------ ------------------- +# ^ | | +# resultsChan | walking errors, | | searchRequestsChan +# | number of files | -----+----- +# ----+--------------------------- | | +# | | (when walking finished) |a path |a path to file +# | | | | +# | | V V +# | | ------------ ------------ +# | | | worker 1 | | worker 2 | +# | | ------------ ------------ +# | | matches in the file | | +# | -------------------------------- | +# | matches in the file | +# ---------------------------------------------- +# +# The matches from each file are passed at once as FileResult type. + +proc worker(initSearchOpt: SearchOpt) {.thread.} = + searchOpt = initSearchOpt # init thread-local var + declareCompiledPatterns(searchOptC, SearchOptComp): + compile1Pattern(searchOpt.pattern, searchOptC.pattern) + searchOptC.inFile.add searchOpt.inFile.compileArray() + searchOptC.notInFile.add searchOpt.notInFile.compileArray() + searchOptC.inContext.add searchOpt.inContext.compileArray() + searchOptC.notInContext.add searchOpt.notInContext.compileArray() + while true: + let (fileNo, filename) = searchRequestsChan.recv() + var fileResult: FileResult + for output in processFile(searchOptC, filename, + yieldContents=(optReplace in options)): + fileResult.add(output) + resultsChan.send((false, fileNo, filename, move(fileResult))) + +proc pathProducer(arg: (seq[string], WalkOpt)) {.thread.} = + let paths = arg[0] + walkOpt = arg[1] # init thread-local copy of opt + var + nextFileN = 0 + for entry in walkRec(paths): + if entry.error == "": + searchRequestsChan.send((nextFileN, entry.filename)) + else: + resultsChan.send((false, nextFileN, entry.filename, + @[Output(kind: openError, msg: entry.error)])) + nextFileN += 1 + resultsChan.send((true, nextFileN, "", @[])) # pass total number of files + +proc runMultiThread() = + var + workers = newSeq[Thread[SearchOpt]](nWorkers) + storage = newTable[int, (string, FileResult) ]() + # file number -> tuple[filename, fileResult - accumulated data structure] + firstUnprocessedFile = 0 # for always processing files in the same order + open(searchRequestsChan) + open(resultsChan) + for n in 0 ..< nWorkers: + createThread(workers[n], worker, searchOpt) + var producerThread: Thread[(seq[string], WalkOpt)] + createThread(producerThread, pathProducer, (paths, walkOpt)) + declareCompiledPatterns(pat, SinglePattern): + compile1Pattern(searchOpt.pattern, pat.pattern) + template add1fileResult(fileNo: int, fname: string, fResult: FileResult) = + storage[fileNo] = (fname, fResult) + while storage.haskey(firstUnprocessedFile): + let fileResult = storage[firstUnprocessedFile][1] + let filename = storage[firstUnprocessedFile][0] + processFileResult(pat.pattern, filename, fileResult) + storage.del(firstUnprocessedFile) + firstUnprocessedFile += 1 + var totalFiles = -1 # will be known when pathProducer finishes + while totalFiles == -1 or firstUnprocessedFile < totalFiles: + let msg = resultsChan.recv() + if msg.finished: + totalFiles = msg.fileNo + else: + add1fileResult(msg.fileNo, msg.filename, msg.fileResult) proc reportError(msg: string) = printError "Error: " & msg @@ -535,6 +1231,15 @@ proc checkOptions(subset: TOptions, a, b: string) = if subset <= options: quit("cannot specify both '$#' and '$#'" % [a, b]) +proc parseNonNegative(str: string, key: string): int = + try: + result = parseInt(str) + except ValueError: + reportError("Option " & key & " requires an integer but '" & + str & "' was given") + if result < 0: + reportError("A positive integer is expected for option " & key) + when defined(posix): useWriteStyled = terminal.isatty(stdout) # that should be before option processing to allow override of useWriteStyled @@ -543,14 +1248,21 @@ for kind, key, val in getopt(): case kind of cmdArgument: if options.contains(optStdin): - filenames.add(key) - elif pattern.len == 0: - pattern = key - elif options.contains(optReplace) and replacement.len == 0: + paths.add(key) + elif not searchOpt.patternSet: + searchOpt.pattern = key + searchOpt.patternSet = true + elif options.contains(optReplace) and not replacementSet: replacement = key + replacementSet = true else: - filenames.add(key) + paths.add(key) of cmdLongOption, cmdShortOption: + proc addNotEmpty(s: var seq[string], name: string) = + if name == "": + reportError("empty string given for option --" & key & + " (did you forget `:`?)") + s.add name case normalize(key) of "find", "f": incl(options, optFind) of "replace", "!": incl(options, optReplace) @@ -569,98 +1281,154 @@ for kind, key, val in getopt(): of "confirm": incl(options, optConfirm) of "stdin": incl(options, optStdin) of "word", "w": incl(options, optWord) - of "ignorecase", "i": incl(options, optIgnoreCase) - of "ignorestyle", "y": incl(options, optIgnoreStyle) - of "ext": extensions.add val.split('|') - of "noext": skipExtensions.add val.split('|') - of "excludedir", "exclude-dir": excludeDir.add rex(val) - of "includefile", "include-file": includeFile.add rex(val) - of "excludefile", "exclude-file": excludeFile.add rex(val) - of "nocolor": useWriteStyled = false + of "ignorecase", "ignore-case", "i": incl(options, optIgnoreCase) + of "ignorestyle", "ignore-style", "y": incl(options, optIgnoreStyle) + of "threads", "j": + if val == "": + nWorkers = countProcessors() + else: + nWorkers = parseNonNegative(val, key) + of "extensions", "ex", "ext": walkOpt.extensions.add val.split('|') + of "nextensions", "notextensions", "nex", "notex", + "noext", "no-ext": # 2 deprecated options + walkOpt.notExtensions.add val.split('|') + of "dirname", "di": + walkOpt.dirname.addNotEmpty val + of "ndirname", "notdirname", "ndi", "notdi", + "excludedir", "exclude-dir", "ed": # 3 deprecated options + walkOpt.notDirname.addNotEmpty val + of "dirpath", "dirp", + "includedir", "include-dir", "id": # 3 deprecated options + walkOpt.dirPath.addNotEmpty val + of "ndirpath", "notdirpath", "ndirp", "notdirp": + walkOpt.notDirPath.addNotEmpty val + of "filename", "fi", + "includefile", "include-file", "if": # 3 deprecated options + walkOpt.filename.addNotEmpty val + of "nfilename", "nfi", "notfilename", "notfi", + "excludefile", "exclude-file", "ef": # 3 deprecated options + walkOpt.notFilename.addNotEmpty val + of "infile", "inf", + "matchfile", "match", "mf": # 3 deprecated options + searchOpt.inFile.addNotEmpty val + of "ninfile", "notinfile", "ninf", "notinf", + "nomatchfile", "nomatch", "nf": # 3 options are deprecated + searchOpt.notInFile.addNotEmpty val + of "incontext", "inc": + searchOpt.inContext.addNotEmpty val + of "nincontext", "notincontext", "ninc", "notinc": + searchOpt.notInContext.addNotEmpty val + of "bin": + case val + of "on": searchOpt.checkBin = biOn + of "off": searchOpt.checkBin = biOff + of "only": searchOpt.checkBin = biOnly + else: reportError("unknown value for --bin") + of "text", "t": searchOpt.checkBin = biOff + of "count": incl(options, optCount) + of "sorttime", "sort-time", "s": + case normalize(val) + of "off": sortTime = false + of "", "on", "asc", "ascending": + sortTime = true + sortTimeOrder = SortOrder.Ascending + of "desc", "descending": + sortTime = true + sortTimeOrder = SortOrder.Descending + else: reportError("invalid value '" & val & "' for --sortTime") + of "nocolor", "no-color": useWriteStyled = false of "color": case val of "auto": discard - of "never", "false": useWriteStyled = false - of "", "always", "true": useWriteStyled = true + of "off", "never", "false": useWriteStyled = false + of "", "on", "always", "true": useWriteStyled = true else: reportError("invalid value '" & val & "' for --color") - of "colortheme": + of "colortheme", "color-theme": colortheme = normalize(val) if colortheme notin ["simple", "bnw", "ack", "gnu"]: reportError("unknown colortheme '" & val & "'") of "beforecontext", "before-context", "b": - try: - linesBefore = parseInt(val) - except ValueError: - reportError("option " & key & " requires an integer but '" & - val & "' was given") + linesBefore = parseNonNegative(val, key) of "aftercontext", "after-context", "a": - try: - linesAfter = parseInt(val) - except ValueError: - reportError("option " & key & " requires an integer but '" & - val & "' was given") + linesAfter = parseNonNegative(val, key) of "context", "c": - try: - linesContext = parseInt(val) - except ValueError: - reportError("option --context requires an integer but '" & - val & "' was given") - of "newline", "l": newLine = true - of "oneline": oneline = true - of "group", "g": oneline = false + linesContext = parseNonNegative(val, key) + of "newline", "l": + newLine = true + # Tabs are aligned automatically for --group, --newLine, --filenames + expandTabs = false + of "group", "g": + oneline = false + expandTabs = false + of "cols", "%": + incl(options, optLimitChars) + termWidth = terminalWidth() + if val == "auto" or key == "%": + limitCharUsr = termWidth + when defined(windows): # Windows cmd & powershell add an empty line + limitCharUsr -= 1 # when printing '\n' right after the last column + elif val == "": + limitCharUsr = 80 + else: + limitCharUsr = parseNonNegative(val, key) + of "onlyascii", "only-ascii", "@": + if val == "" or val == "on" or key == "@": + optOnlyAscii = true + elif val == "off": + optOnlyAscii = false + else: + printError("unknown value for --onlyAscii option") of "verbose": incl(options, optVerbose) - of "filenames": incl(options, optFilenames) + of "filenames": + incl(options, optFilenames) + expandTabs = false of "help", "h": writeHelp() of "version", "v": writeVersion() + of "": incl(options, optPipe) else: reportError("unrecognized option '" & key & "'") of cmdEnd: assert(false) # cannot happen checkOptions({optFind, optReplace}, "find", "replace") +checkOptions({optCount, optReplace}, "count", "replace") checkOptions({optPeg, optRegex}, "peg", "re") checkOptions({optIgnoreCase, optIgnoreStyle}, "ignore_case", "ignore_style") checkOptions({optFilenames, optReplace}, "filenames", "replace") +checkOptions({optPipe, optStdin}, "-", "stdin") +checkOptions({optPipe, optFilenames}, "-", "filenames") +checkOptions({optPipe, optConfirm}, "-", "confirm") +checkOptions({optPipe, optRecursive}, "-", "recursive") linesBefore = max(linesBefore, linesContext) linesAfter = max(linesAfter, linesContext) +if optPipe in options and paths.len != 0: + reportError("both - and paths are specified") + if optStdin in options: - pattern = ask("pattern [ENTER to exit]: ") - if pattern.len == 0: quit(0) + searchOpt.pattern = ask("pattern [ENTER to exit]: ") + if searchOpt.pattern.len == 0: quit(0) if optReplace in options: replacement = ask("replacement [supports $1, $# notations]: ") -if pattern.len == 0: +if optReplace in options and not replacementSet: + reportError("provide REPLACEMENT as second argument (use \"\" for empty one)") +if optReplace in options and paths.len == 0 and optPipe notin options: + reportError("provide paths for replacement explicitly (use . for current directory)") + +if searchOpt.pattern == "" and optFilenames notin options: reportError("empty pattern was given") else: - var counter = 0 - var errors = 0 - if filenames.len == 0: - filenames.add(os.getCurrentDir()) - if optRegex notin options: - if optWord in options: - pattern = r"(^ / !\letter)(" & pattern & r") !\letter" - if optIgnoreStyle in options: - pattern = "\\y " & pattern - elif optIgnoreCase in options: - pattern = "\\i " & pattern - let pegp = peg(pattern) - for f in items(filenames): - walker(pegp, f, counter, errors) + if paths.len == 0 and optPipe notin options: + paths.add(".") + if optPipe in options or nWorkers == 0: + run1Thread() else: - var reflags = {reStudy} - if optIgnoreStyle in options: - pattern = styleInsensitive(pattern) - if optWord in options: - # see https://github.com/nim-lang/Nim/issues/13528#issuecomment-592786443 - pattern = r"(^|\W)(:?" & pattern & r")($|\W)" - if {optIgnoreCase, optIgnoreStyle} * options != {}: - reflags.incl reIgnoreCase - let rep = if optRex in options: rex(pattern, reflags) - else: re(pattern, reflags) - for f in items(filenames): - walker(rep, f, counter, errors) - if errors != 0: - printError $errors & " errors" - stdout.write($counter & " matches\n") - if errors != 0: + runMultiThread() + if gVar.errors != 0: + printError $gVar.errors & " errors" + if searchOpt.pattern != "": + # PATTERN allowed to be empty if --filenames is given + printBold($gVar.matches & " matches") + stdout.write("\n") + if gVar.errors != 0: quit(1) diff --git a/tools/nimgrep.nim.cfg b/tools/nimgrep.nim.cfg deleted file mode 100644 index 6d0ea5aad..000000000 --- a/tools/nimgrep.nim.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# The GC is stable enough now: - -#--gc:none - - diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index 5d532a0fd..063a02779 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateBuildShellScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated from niminst\n" & +# result = "#!/bin/sh\n# Generated from niminst\n" & # "# Template is in tools/niminst/buildsh.nimf\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" @@ -21,10 +21,23 @@ do optosname=$2 shift 2 ;; + --parallel) + parallel=$2 + shift 2 + ;; --extraBuildArgs) extraBuildArgs=" $2" shift 2 ;; + -h | --help) + echo "Options:" + echo " --os <OS>" + echo " --cpu <CPU architecture>" + echo " --osname <name> Additional OS specification (used for Android)" + echo " --extraBuildArgs <args> Additional arguments passed to the compiler" + echo " --parallel <number> Multiprocess build. Requires GNU parallel" + exit 0 + ;; --) # End of all options shift break; @@ -39,7 +52,15 @@ do esac done +parallel="${parallel:-0}" CC="${CC:-gcc}" +if [ "$parallel" -gt 1 ]; then + if ! command -v sem > /dev/null; then + echo "Error: GNU parallel is required to use --parallel" + exit 1 + fi + CC="sem -j $parallel --id $$ ${CC}" +fi COMP_FLAGS="${CPPFLAGS:-} ${CFLAGS:-} ?{c.ccompiler.flags}$extraBuildArgs" LINK_FLAGS="${LDFLAGS:-} ?{c.linker.flags}" PS4="" @@ -88,6 +109,11 @@ case $uos in CC="clang" LINK_FLAGS="$LINK_FLAGS -lm" ;; + *crossos* ) + myos="crossos" + CC="clang" + LINK_FLAGS="$LINK_FLAGS -lm" + ;; *openbsd* ) myos="openbsd" CC="clang" @@ -96,6 +122,7 @@ case $uos in *netbsd* ) myos="netbsd" LINK_FLAGS="$LINK_FLAGS -lm" + ucpu=`uname -p` ;; *darwin* ) myos="macosx" @@ -113,6 +140,11 @@ case $uos in myos="solaris" LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" ;; + *SunOS* ) + myos="solaris" + LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" + isOpenIndiana="yes" + ;; *haiku* ) myos="haiku" LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" @@ -133,7 +165,12 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - mycpu="i386" ;; + if [ "$isOpenIndiana" = "yes" ] || [ `uname -o` == "illumos" ] ; then + mycpu="amd64" + else + mycpu="i386" + fi + ;; *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; *sparc*|*sun* ) @@ -156,10 +193,16 @@ case $ucpu in mycpu="powerpc64" ;; *power*|*ppc* ) if [ "$myos" = "freebsd" ] ; then - COMP_FLAGS="$COMP_FLAGS -m64" - LINK_FLAGS="$LINK_FLAGS -m64" + if [ "$ucpu" != "powerpc" ] ; then + COMP_FLAGS="$COMP_FLAGS -m64" + LINK_FLAGS="$LINK_FLAGS -m64" + fi mycpu=`uname -p` - else + case $mycpu in + powerpc64le) + mycpu="powerpc64el" + esac + else mycpu="powerpc" fi ;; @@ -183,10 +226,14 @@ case $ucpu in mycpu="alpha" ;; *aarch64*|*arm64* ) mycpu="arm64" ;; - *arm*|*armv6l*|*armv71* ) + *arm*|*armv6l*|*armv7l*|*armv8l* ) mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; + *e2k* ) + mycpu="e2k" ;; + *loongarch64* ) + mycpu="loongarch64" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 @@ -218,6 +265,9 @@ case $myos in $CC $COMP_FLAGS -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} # add(linkCmd, " \\\n" & changeFileExt(f, "o")) # end for + if [ "$parallel" -gt 0 ]; then + sem --wait --id $$ + fi $CC -o ?{"$binDir/" & toLowerAscii(c.name)} ?linkCmd $LINK_FLAGS ;; # end for diff --git a/tools/niminst/debcreation.nim b/tools/niminst/debcreation.nim index d0f46fa52..219cb44ce 100644 --- a/tools/niminst/debcreation.nim +++ b/tools/niminst/debcreation.nim @@ -9,6 +9,10 @@ import osproc, times, os, strutils + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + # http://www.debian.org/doc/manuals/maint-guide/ # Required files for debhelper. diff --git a/tools/niminst/deinstall.nimf b/tools/niminst/deinstall.nimf index 8b4477369..0dcfda75e 100644 --- a/tools/niminst/deinstall.nimf +++ b/tools/niminst/deinstall.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateDeinstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii if [ $# -eq 1 ] ; then diff --git a/tools/niminst/install.nimf b/tools/niminst/install.nimf index a78914ecd..75ff9ce11 100644 --- a/tools/niminst/install.nimf +++ b/tools/niminst/install.nimf @@ -1,24 +1,24 @@ #? stdtmpl(subsChar = '?') | standard #proc generateInstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii set -e if [ $# -eq 1 ] ; then -# if c.cat[fcUnixBin].len > 0: - if test -f ?{c.cat[fcUnixBin][0].toUnix} +#if c.cat[fcUnixBin].len > 0: + if [ -f "?{c.cat[fcUnixBin][0].toUnix}" ] then echo "?c.displayName build detected" else echo "Please build ?c.displayName before installing it" exit 1 fi -# end if +#end if case $1 in "--help"|"-h"|"help"|"h") echo "?c.displayName installation script" - echo "Usage: [sudo] sh install.sh DIR" + echo "Usage: [sudo] [env DESTDIR=...] sh install.sh DIR" echo "Where DIR may be:" echo " /usr/bin" echo " /usr/local/bin" @@ -29,19 +29,19 @@ if [ $# -eq 1 ] ; then exit 1 ;; "/usr/bin") - bindir=/usr/bin - configdir=/etc/?proj - libdir=/usr/lib/?proj - docdir=/usr/share/?proj/doc - datadir=/usr/share/?proj/data + bindir=$1 + configdir="/etc/?proj" + libdir="/usr/lib/?proj" + docdir="/usr/share/?proj/doc" + datadir="/usr/share/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/usr/local/bin") - bindir=/usr/local/bin - configdir=/etc/?proj - libdir=/usr/local/lib/?proj - docdir=/usr/local/share/?proj/doc - datadir=/usr/local/share/?proj/data + bindir=$1 + configdir="/etc/?proj" + libdir="/usr/local/lib/?proj" + docdir="/usr/local/share/?proj/doc" + datadir="/usr/local/share/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/opt") @@ -51,9 +51,6 @@ if [ $# -eq 1 ] ; then docdir="/opt/?proj/doc" datadir="/opt/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" - mkdir -p /opt/?proj - mkdir -p $bindir - mkdir -p $configdir ;; *) bindir="$1/?proj/bin" @@ -62,16 +59,22 @@ if [ $# -eq 1 ] ; then docdir="$1/?proj/doc" datadir="$1/?proj/data" nimbleDir="$1/?proj" - mkdir -p $1/?proj - mkdir -p $bindir - mkdir -p $configdir ;; esac - mkdir -p $libdir - mkdir -p $docdir - mkdir -p $configdir - mkdir -p $nimbleDir/ + bindir="${DESTDIR}${bindir}" + configdir="${DESTDIR}${configdir}" + libdir="${DESTDIR}${libdir}" + docdir="${DESTDIR}${docdir}" + datadir="${DESTDIR}${datadir}" + nimbleDir="${DESTDIR}${nimbleDir}" + + mkdir -p "$bindir" + mkdir -p "$configdir" + mkdir -p "$libdir" + mkdir -p "$docdir" + mkdir -p "$datadir" + mkdir -p "$nimbleDir" echo "copying files..." #var createdDirs = newStringTable() #for cat in {fcConfig..fcLib, fcNimble}: @@ -84,46 +87,46 @@ if [ $# -eq 1 ] ; then # end if # if mk.len > 0 and not createdDirs.hasKey(mk): # createdDirs[mk] = "true" - mkdir -p ?{mk.toUnix} + mkdir -p "?{mk.toUnix}" # end if # end for #end for #for f in items(c.cat[fcUnixBin]): - cp ?f.toUnix $bindir/?f.skipRoot.toUnix - chmod 755 $bindir/?f.skipRoot.toUnix + cp "?f.toUnix" "$bindir/?f.skipRoot.toUnix" + chmod 755 "$bindir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcConfig]): - cp ?f.toUnix $configdir/?f.skipRoot.toUnix - chmod 644 $configdir/?f.skipRoot.toUnix + cp "?f.toUnix" "$configdir/?f.skipRoot.toUnix" + chmod 644 "$configdir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcData]): - if [ -f ?f.toUnix ]; then - cp ?f.toUnix $datadir/?f.skipRoot.toUnix - chmod 644 $datadir/?f.skipRoot.toUnix + if [ -f "?f.toUnix" ]; then + cp "?f.toUnix" "$datadir/?f.skipRoot.toUnix" + chmod 644 "$datadir/?f.skipRoot.toUnix" fi #end for #for f in items(c.cat[fcDoc]): - if [ -f ?f.toUnix ]; then - cp ?f.toUnix $docdir/?f.skipRoot.toUnix - chmod 644 $docdir/?f.skipRoot.toUnix + if [ -f "?f.toUnix" ]; then + cp "?f.toUnix" "$docdir/?f.skipRoot.toUnix" + chmod 644 "$docdir/?f.skipRoot.toUnix" fi #end for #for f in items(c.cat[fcLib]): - cp ?f.toUnix $libdir/?f.skipRoot.toUnix - chmod 644 $libdir/?f.skipRoot.toUnix + cp "?f.toUnix" "$libdir/?f.skipRoot.toUnix" + chmod 644 "$libdir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcNimble]): - cp ?f.toUnix $nimbleDir/?f.toUnix - chmod 644 $nimbleDir/?f.toUnix + cp "?f.toUnix" "$nimbleDir/?f.toUnix" + chmod 644 "$nimbleDir/?f.toUnix" #end for -cp ?{c.nimblePkgName}.nimble $nimbleDir/?{c.nimblePkgName}.nimble -chmod 644 $nimbleDir/?{c.nimblePkgName}.nimble +cp "?{c.nimblePkgName}.nimble" "$nimbleDir/?{c.nimblePkgName}.nimble" +chmod 644 "$nimbleDir/?{c.nimblePkgName}.nimble" echo "installation successful" else echo "?c.displayName installation script" - echo "Usage: [sudo] sh install.sh DIR" + echo "Usage: [sudo] [env DESTDIR=...] sh install.sh DIR" echo "Where DIR may be:" echo " /usr/bin" echo " /usr/local/bin" diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index ad9d55c03..002bc0592 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -16,6 +16,7 @@ endif target := ?{"$(binDir)/" & toLowerAscii(c.name)} + ucpu := $(shell sh -c 'uname -m | tr "[:upper:]" "[:lower:]"') ifeq ($(OS),Windows_NT) uos := windows @@ -25,7 +26,8 @@ endif ifeq ($(uos),linux) myos = linux - LDFLAGS += -ldl -lm + ## add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17 + LDFLAGS += -ldl -lm -lrt endif ifeq ($(uos),dragonfly) myos = freebsd @@ -43,6 +45,7 @@ endif ifeq ($(uos),netbsd) myos = netbsd LDFLAGS += -lm + ucpu = $(shell sh -c 'uname -p') endif ifeq ($(uos),darwin) myos = macosx @@ -102,9 +105,18 @@ endif ifeq ($(ucpu),x86_64) mycpu = amd64 endif +ifeq ($(ucpu),parisc64) + mycpu = hppa +endif +ifeq ($(ucpu),s390x) + mycpu = s390x +endif ifeq ($(ucpu),sparc) mycpu = sparc endif +ifeq ($(ucpu),sparc64) + mycpu = sparc64 +endif ifeq ($(ucpu),sun) mycpu = sparc endif @@ -124,10 +136,13 @@ ifeq ($(ucpu),powerpc) mycpu = $(shell sh -c 'uname -p | tr "[:upper:]" "[:lower:]"') CFLAGS += -m64 LDFLAGS += -m64 + ifeq ($(mycpu),powerpc64le) + mycpu = powerpc64el + endif endif endif ifeq ($(ucpu),ppc) - mycpu = ppc + mycpu = powerpc endif ifneq (,$(filter $(ucpu), mips mips64)) mycpu = $(shell /bin/sh -c '"$(CC)" -dumpmachine | sed "s/-.*//"') @@ -153,14 +168,26 @@ endif ifeq ($(ucpu),armv7hl) mycpu = arm endif +ifeq ($(ucpu),armv8l) + mycpu = arm +endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif +ifeq ($(ucpu),arm64) + mycpu = arm64 +endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif +ifeq ($(ucpu),e2k) + mycpu = e2k +endif +ifeq ($(ucpu),loongarch64) + mycpu = loongarch64 +endif ifndef mycpu - $(error unknown processor: $(ucpu)) + $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif # for osA in 1..c.oses.len: diff --git a/tools/niminst/nim-file.ico b/tools/niminst/nim-file.ico new file mode 100644 index 000000000..0685fedcb --- /dev/null +++ b/tools/niminst/nim-file.ico Binary files differdiff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index a5fc388cd..40ee79814 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -7,15 +7,16 @@ # distribution, for details about the copyright. # -const - haveZipLib = false # zip not in stdlib anymore +import + os, strutils, parseopt, parsecfg, strtabs, streams, debcreation -when haveZipLib: - import zipfiles +import ../../dist/checksums/src/checksums/sha1 -import - os, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - std / sha1 +when defined(nimPreviewSlimSystem): + import std/syncio + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} const maxOS = 20 # max number of OSes @@ -36,7 +37,7 @@ type actionInno, # action: create Inno Setup installer actionNsis, # action: create NSIS installer actionScripts # action: create install and deinstall scripts - actionZip, # action: create zip file + actionZip # action: create zip file actionXz, # action: create xz file actionDeb # action: prepare deb package @@ -75,7 +76,7 @@ const "$configdir", "$datadir", "$docdir", "$libdir" ] -proc iniConfigData(c: var ConfigData) = +func iniConfigData(c: var ConfigData) = c.actions = {} for i in low(FileCategory)..high(FileCategory): c.cat[i] = @[] c.binPaths = @[] @@ -107,17 +108,17 @@ proc iniConfigData(c: var ConfigData) = c.debOpts.shortDesc = "" c.debOpts.licenses = @[] -proc firstBinPath(c: ConfigData): string = +func firstBinPath(c: ConfigData): string = if c.binPaths.len > 0: result = c.binPaths[0] else: result = "" -proc `\`(a, b: string): string = +func `\`(a, b: string): string = result = if a.len == 0: b else: a & '\\' & b template toUnix(s: string): string = s.replace('\\', '/') template toWin(s: string): string = s.replace('/', '\\') -proc skipRoot(f: string): string = +func skipRoot(f: string): string = # "abc/def/xyz" --> "def/xyz" var i = 0 result = "" @@ -167,11 +168,11 @@ proc parseCmdLine(c: var ConfigData) = next(p) var kind = p.kind var key = p.key - var val = p.val.string + var val = p.val case kind of cmdArgument: if c.actions == {}: - for a in split(normalize(key.string), {';', ','}): + for a in split(normalize(key), {';', ','}): case a of "csource": incl(c.actions, actionCSource) of "scripts": incl(c.actions, actionScripts) @@ -182,11 +183,11 @@ proc parseCmdLine(c: var ConfigData) = of "deb": incl(c.actions, actionDeb) else: quit(Usage) else: - c.infile = addFileExt(key.string, "ini") - c.nimArgs = cmdLineRest(p).string + c.infile = addFileExt(key, "ini") + c.nimArgs = cmdLineRest(p) break of cmdLongOption, cmdShortOption: - case normalize(key.string) + case normalize(key) of "help", "h": stdout.write(Usage) quit(0) @@ -204,7 +205,7 @@ proc parseCmdLine(c: var ConfigData) = if c.infile.len == 0: quit(Usage) if c.mainfile.len == 0: c.mainfile = changeFileExt(c.infile, "nim") -proc eqT(a, b: string; t: proc (a: char): char{.nimcall.}): bool = +proc eqT(a, b: string; t: proc (a: char): char {.nimcall.}): bool {.effectsOf: t.} = ## equality under a transformation ``t``. candidate for the stdlib? var i = 0 var j = 0 @@ -221,11 +222,11 @@ proc eqT(a, b: string; t: proc (a: char): char{.nimcall.}): bool = inc j result = i >= a.len and j >= b.len -proc tPath(c: char): char = +func tPath(c: char): char = if c == '\\': '/' else: c -proc ignoreFile(f, explicit: string, allowHtml: bool): bool = +func ignoreFile(f, explicit: string, allowHtml: bool): bool = let (_, name, ext) = splitFile(f) let html = if not allowHtml: ".html" else: "" result = (ext in ["", ".exe", ".idx", ".o", ".obj", ".dylib"] or @@ -283,13 +284,13 @@ proc yesno(p: var CfgParser, v: string): bool = result = false else: quit(errorStr(p, "unknown value; use: yes|no")) -proc incl(s: var seq[string], x: string): int = +func incl(s: var seq[string], x: string): int = for i in 0 ..< s.len: if cmpIgnoreStyle(s[i], x) == 0: return i s.add(x) result = s.len-1 -proc platforms(c: var ConfigData, v: string) = +func platforms(c: var ConfigData, v: string) = for line in splitLines(v): let p = line.find(": ") if p <= 1: continue @@ -462,10 +463,10 @@ proc readCFiles(c: var ConfigData, osA, cpuA: int) = else: quit("Cannot open: " & f) -proc buildDir(os, cpu: int): string = - return "c_code" / ($os & "_" & $cpu) +func buildDir(os, cpu: int): string = + "c_code" / ($os & "_" & $cpu) -proc getOutputDir(c: var ConfigData): string = +func getOutputDir(c: var ConfigData): string = if c.outdir.len > 0: c.outdir else: "build" proc writeFile(filename, content, newline: string) = @@ -483,8 +484,6 @@ proc deduplicateFiles(c: var ConfigData) = let build = getOutputDir(c) for osA in countup(1, c.oses.len): for cpuA in countup(1, c.cpus.len): - when not defined(nimNoNilSeqs): - if c.cfiles[osA][cpuA].isNil: c.cfiles[osA][cpuA] = @[] if c.explicitPlatforms and not c.platforms[osA][cpuA]: continue for dup in mitems(c.cfiles[osA][cpuA]): let key = $secureHashFile(build / dup) @@ -516,6 +515,17 @@ template gatherFiles(fun, libpath, outDir) = # commenting out for now, see discussion in https://github.com/nim-lang/Nim/pull/13413 # copySrc(libpath / "lib/wrappers/linenoise/linenoise.h") +proc exe(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') + +proc findNim(): string = + let nim = "nim".exe + result = quoteShell("bin" / nim) + if not fileExists(result): + result = "nim" + proc srcdist(c: var ConfigData) = let cCodeDir = getOutputDir(c) / "c_code" if not dirExists(cCodeDir): createDir(cCodeDir) @@ -534,10 +544,10 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("nim compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --incremental:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % - [osname, cpuname, c.nimArgs, c.mainfile] + [findNim(), osname, cpuname, c.nimArgs, c.mainfile] echo(cmd) if execShellCmd(cmd) != 0: quit("Error: call to nim compiler failed") @@ -596,42 +606,6 @@ proc setupDist2(c: var ConfigData) = else: quit("External program failed") -# ------------------ generate ZIP file --------------------------------------- -when haveZipLib: - proc zipDist(c: var ConfigData) = - var proj = toLowerAscii(c.name) & "-" & c.version - var n = "$#.zip" % proj - if c.outdir.len == 0: n = "build" / n - else: n = c.outdir / n - var z: ZipArchive - if open(z, n, fmWrite): - addFile(z, proj / buildBatFile, "build" / buildBatFile) - addFile(z, proj / buildBatFile32, "build" / buildBatFile32) - addFile(z, proj / buildBatFile64, "build" / buildBatFile64) - addFile(z, proj / buildShFile, "build" / buildShFile) - addFile(z, proj / makeFile, "build" / makeFile) - addFile(z, proj / installShFile, installShFile) - addFile(z, proj / deinstallShFile, deinstallShFile) - - template addFileAux(src, dst) = addFile(z, dst, src) - gatherFiles(addFileAux, c.libpath, proj / "c_code") - for osA in 1..c.oses.len: - for cpuA in 1..c.cpus.len: - var dir = buildDir(osA, cpuA) - for k, f in walkDir("build" / dir): - if k == pcFile: addFile(z, proj / dir / extractFilename(f), f) - - for cat in items({fcConfig..fcOther, fcUnix, fcNimble}): - for f in items(c.cat[cat]): addFile(z, proj / f, f) - - # Copy the .nimble file over - let nimbleFile = c.nimblePkgName & ".nimble" - processFile(z, proj / nimbleFile, nimbleFile) - - close(z) - else: - quit("Cannot open for writing: " & n) - proc xzDist(c: var ConfigData; windowsZip=false) = let proj = toLowerAscii(c.name) & "-" & c.version let tmpDir = if c.outdir.len == 0: "build" else: c.outdir @@ -647,7 +621,7 @@ proc xzDist(c: var ConfigData; windowsZip=false) = if not windowsZip and not fileExists("build" / buildBatFile): quit("No C sources found in ./build/, please build by running " & - "./koch csource -d:release.") + "./koch csource -d:danger.") if not windowsZip: processFile(proj / buildBatFile, "build" / buildBatFile) @@ -753,21 +727,25 @@ proc debDist(c: var ConfigData) = # ------------------- main ---------------------------------------------------- -var c: ConfigData -iniConfigData(c) -parseCmdLine(c) -parseIniFile(c) -if actionInno in c.actions: - setupDist(c) -if actionNsis in c.actions: - setupDist2(c) -if actionCSource in c.actions: - srcdist(c) -if actionScripts in c.actions: - writeInstallScripts(c) -if actionZip in c.actions: - xzDist(c, true) -if actionXz in c.actions: - xzDist(c) -if actionDeb in c.actions: - debDist(c) +proc main() = + var c: ConfigData + iniConfigData(c) + parseCmdLine(c) + parseIniFile(c) + if actionInno in c.actions: + setupDist(c) + if actionNsis in c.actions: + setupDist2(c) + if actionCSource in c.actions: + srcdist(c) + if actionScripts in c.actions: + writeInstallScripts(c) + if actionZip in c.actions: + xzDist(c, true) + if actionXz in c.actions: + xzDist(c) + if actionDeb in c.actions: + debDist(c) + +when isMainModule: + main() diff --git a/tools/niminst/setup.ico b/tools/niminst/setup.ico new file mode 100644 index 000000000..867163046 --- /dev/null +++ b/tools/niminst/setup.ico Binary files differdiff --git a/tools/niminst/uninstall.ico b/tools/niminst/uninstall.ico new file mode 100644 index 000000000..aff054644 --- /dev/null +++ b/tools/niminst/uninstall.ico Binary files differdiff --git a/tools/nimrepl.nim b/tools/nimrepl.nim deleted file mode 100644 index ac82d8b75..000000000 --- a/tools/nimrepl.nim +++ /dev/null @@ -1,172 +0,0 @@ -# -# -# Nim REPL -# (c) Copyright 2012 Dominik Picheta -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import glib2, gtk2, gdk2, os, osproc, dialogs, strutils - -when defined(tinyc): - const runCmd = "run" -else: - const runCmd = "c -r" - -var nimExe = findExe("nim") -if nimExe.len == 0: nimExe = "../bin" / addFileExt("nim", os.ExeExt) - -proc execCode(code: string): string = - var f: File - if open(f, "temp.nim", fmWrite): - f.write(code) - f.close() - result = osproc.execProcess( - "$# $# --verbosity:0 --hint[Conf]:off temp.nim" % [nimExe, runCmd], - options = {poStdErrToStdOut}) - else: - result = "cannot open file 'temp.nim'" - -var shiftPressed = false -var w: gtk2.PWindow -var inputTextBuffer: PTextBuffer -var outputTextBuffer: PTextBuffer - -proc destroy(widget: PWidget, data: pointer){.cdecl.} = - main_quit() - -proc fileOpenClicked(menuitem: PMenuItem, userdata: pointer) {.cdecl.} = - var path = chooseFileToOpen(w) - if path != "": - var file = readFile(path) - if file != nil: - set_text(inputTextBuffer, file, len(file).gint) - else: - error(w, "Unable to read from file") - -proc fileSaveClicked(menuitem: PMenuItem, userdata: pointer) {.cdecl.} = - var path = chooseFileToSave(w) - - if path == "": return - var startIter: TTextIter - var endIter: TTextIter - get_start_iter(inputTextBuffer, addr(startIter)) - get_end_iter(inputTextBuffer, addr(endIter)) - var inputText = get_text(inputTextBuffer, addr(startIter), - addr(endIter), false) - var f: File - if open(f, path, fmWrite): - f.write(inputText) - f.close() - else: - error(w, "Unable to write to file") - -proc inputKeyPressed(widget: PWidget, event: PEventKey, - userdata: pointer): bool {.cdecl.} = - if ($keyval_name(event.keyval)).tolower() == "shift_l": - # SHIFT is pressed - shiftPressed = true - -proc setError(msg: string) = - outputTextBuffer.setText(msg, msg.len.gint) - -proc inputKeyReleased(widget: PWidget, event: PEventKey, - userdata: pointer): bool {.cdecl.} = - #echo(keyval_name(event.keyval)) - if ($keyval_name(event.keyval)).tolower() == "shift_l": - # SHIFT is released - shiftPressed = false - - if ($keyval_name(event.keyval)).tolower() == "return": - #echo($keyval_name(event.keyval), "Shift_L") - # Enter pressed - if not shiftPressed: - var startIter: TTextIter - var endIter: TTextIter - get_start_iter(inputTextBuffer, addr(startIter)) - get_end_iter(inputTextBuffer, addr(endIter)) - var inputText = get_text(inputTextBuffer, addr(startIter), - addr(endIter), false) - - try: - var r = execCode($inputText) - set_text(outputTextBuffer, r, len(r).gint) - except IOError: - setError("Error: Could not open file temp.nim") - - -proc initControls() = - w = window_new(gtk2.WINDOW_TOPLEVEL) - set_default_size(w, 500, 600) - set_title(w, "Nim REPL") - discard signal_connect(w, "destroy", SIGNAL_FUNC(nimrepl.destroy), nil) - - # MainBox (vbox) - var mainBox = vbox_new(false, 0) - add(w, mainBox) - - # TopMenu (MenuBar) - var topMenu = menu_bar_new() - show(topMenu) - - var fileMenu = menu_new() - var openMenuItem = menu_item_new("Open") - append(fileMenu, openMenuItem) - show(openMenuItem) - discard signal_connect(openMenuItem, "activate", - SIGNAL_FUNC(fileOpenClicked), nil) - var saveMenuItem = menu_item_new("Save...") - append(fileMenu, saveMenuItem) - show(saveMenuItem) - discard signal_connect(saveMenuItem, "activate", - SIGNAL_FUNC(fileSaveClicked), nil) - var fileMenuItem = menu_item_new("File") - - - set_submenu(fileMenuItem, fileMenu) - show(fileMenuItem) - append(topMenu, fileMenuItem) - - pack_start(mainBox, topMenu, false, false, 0) - - # VPaned - Separates the InputTextView and the OutputTextView - var paned = vpaned_new() - set_position(paned, 450) - pack_start(mainBox, paned, true, true, 0) - show(paned) - - # Init the TextBuffers - inputTextBuffer = text_buffer_new(nil) - outputTextBuffer = text_buffer_new(nil) - - # InputTextView (TextView) - var inputScrolledWindow = scrolled_window_new(nil, nil) - set_policy(inputScrolledWindow, POLICY_AUTOMATIC, POLICY_AUTOMATIC) - var inputTextView = text_view_new(inputTextBuffer) - add_with_viewport(inputScrolledWindow, inputTextView) - add1(paned, inputScrolledWindow) - show(inputScrolledWindow) - show(inputTextView) - - discard signal_connect(inputTextView, "key-release-event", - SIGNAL_FUNC(inputKeyReleased), nil) - discard signal_connect(inputTextView, "key-press-event", - SIGNAL_FUNC(inputKeyPressed), nil) - - # OutputTextView (TextView) - var outputScrolledWindow = scrolled_window_new(nil, nil) - set_policy(outputScrolledWindow, POLICY_AUTOMATIC, POLICY_AUTOMATIC) - var outputTextView = text_view_new(outputTextBuffer) - add_with_viewport(outputScrolledWindow, outputTextView) - add2(paned, outputScrolledWindow) - show(outputScrolledWindow) - show(outputTextView) - - show(w) - show(mainBox) - -nim_init() -initControls() -main() - diff --git a/tools/nimweb.nim b/tools/nimweb.nim deleted file mode 100644 index 10319be87..000000000 --- a/tools/nimweb.nim +++ /dev/null @@ -1,556 +0,0 @@ -# -# -# Nim Website Generator -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import - os, strutils, times, parseopt, parsecfg, streams, strtabs, tables, - re, htmlgen, macros, md5, osproc, parsecsv, algorithm - -from xmltree import escape - -type - TKeyValPair = tuple[key, id, val: string] - TConfigData = object of RootObj - tabs, links: seq[TKeyValPair] - doc, srcdoc, srcdoc2, webdoc, pdf: seq[string] - authors, projectName, projectTitle, logo, infile, ticker: string - vars: StringTableRef - nimCompiler: string - nimArgs: string - gitURL: string - docHTMLOutput: string - webUploadOutput: string - quotations: Table[string, tuple[quote, author: string]] - numProcessors: int # Set by parallelBuild:n, only works for values > 0. - gaId: string # google analytics ID, nil means analytics are disabled - TRssItem = object - year, month, day, title, url, content: string - TAction = enum - actAll, actOnlyWebsite, actPdf, actJson2, actOnlyDocs - - Sponsor = object - logo: string - name: string - url: string - thisMonth: int - allTime: int - since: string - level: int - -var action: TAction - -proc initConfigData(c: var TConfigData) = - c.tabs = @[] - c.links = @[] - c.doc = @[] - c.srcdoc = @[] - c.srcdoc2 = @[] - c.webdoc = @[] - c.pdf = @[] - c.infile = "" - c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot " - c.gitURL = "https://github.com/nim-lang/Nim" - c.docHTMLOutput = "doc/html" - c.webUploadOutput = "web/upload" - c.authors = "" - c.projectTitle = "" - c.projectName = "" - c.logo = "" - c.ticker = "" - c.vars = newStringTable(modeStyleInsensitive) - c.numProcessors = countProcessors() - # Attempts to obtain the git current commit. - when false: - let (output, code) = execCmdEx("git log -n 1 --format=%H") - if code == 0 and output.strip.len == 40: - c.gitCommit = output.strip - c.quotations = initTable[string, tuple[quote, author: string]]() - -include "website.nimf" - -# ------------------------- configuration file ------------------------------- - -const - version = "0.8" - usage = "nimweb - Nim Website Generator Version " & version & """ - - (c) 2015 Andreas Rumpf -Usage: - nimweb [options] ini-file[.ini] [compile_options] -Options: - -h, --help shows this help - -v, --version shows the version - -o, --output overrides output directory instead of default - web/upload and doc/html - --nimCompiler overrides nim compiler; default = bin/nim - --var:name=value set the value of a variable - --website only build the website, not the full documentation - --pdf build the PDF version of the documentation - --json2 build JSON of the documentation - --onlyDocs build only the documentation - --git.url override base url in generated doc links - --git.commit override commit/branch in generated doc links 'source' - --git.devel override devel branch in generated doc links 'edit' -Compile_options: - will be passed to the Nim compiler -""" - - rYearMonthDay = r"on\s+(\d{2})\/(\d{2})\/(\d{4})" - rssUrl = "http://nim-lang.org/news.xml" - rssNewsUrl = "http://nim-lang.org/news.html" - activeSponsors = "web/sponsors.csv" - inactiveSponsors = "web/inactive_sponsors.csv" - validAnchorCharacters = Letters + Digits - - -macro id(e: untyped): untyped = - ## generates the rss xml ``id`` element. - let e = callsite() - result = xmlCheckedTag(e, "id") - -macro updated(e: varargs[untyped]): untyped = - ## generates the rss xml ``updated`` element. - let e = callsite() - result = xmlCheckedTag(e, "updated") - -proc updatedDate(year, month, day: string): string = - ## wrapper around the update macro with easy input. - result = updated("$1-$2-$3T00:00:00Z" % [year, - repeat("0", 2 - len(month)) & month, - repeat("0", 2 - len(day)) & day]) - -macro entry(e: varargs[untyped]): untyped = - ## generates the rss xml ``entry`` element. - let e = callsite() - result = xmlCheckedTag(e, "entry") - -macro content(e: varargs[untyped]): untyped = - ## generates the rss xml ``content`` element. - let e = callsite() - result = xmlCheckedTag(e, "content", reqAttr = "type") - -proc parseCmdLine(c: var TConfigData) = - var p = initOptParser() - while true: - next(p) - var kind = p.kind - var key = p.key - var val = p.val - case kind - of cmdArgument: - c.infile = addFileExt(key, "ini") - c.nimArgs.add(cmdLineRest(p)) - break - of cmdLongOption, cmdShortOption: - case normalize(key) - of "help", "h": - stdout.write(usage) - quit(0) - of "version", "v": - stdout.write(version & "\n") - quit(0) - of "output", "o": - c.webUploadOutput = val - c.docHTMLOutput = val / "docs" - of "nimcompiler": - c.nimCompiler = val - of "parallelbuild": - try: - let num = parseInt(val) - if num != 0: c.numProcessors = num - except ValueError: - quit("invalid numeric value for --parallelBuild") - of "var": - var idx = val.find('=') - if idx < 0: quit("invalid command line") - c.vars[substr(val, 0, idx-1)] = substr(val, idx+1) - of "website": action = actOnlyWebsite - of "pdf": action = actPdf - of "json2": action = actJson2 - of "onlydocs": action = actOnlyDocs - of "googleanalytics": - c.gaId = val - c.nimArgs.add("--doc.googleAnalytics:" & val & " ") - of "git.url": - c.gitURL = val - of "git.commit": - c.nimArgs.add("--git.commit:" & val & " ") - of "git.devel": - c.nimArgs.add("--git.devel:" & val & " ") - else: - echo("Invalid argument '$1'" % [key]) - quit(usage) - of cmdEnd: break - if c.infile.len == 0: quit(usage) - -proc walkDirRecursively(s: var seq[string], root, ext: string) = - for k, f in walkDir(root): - case k - of pcFile, pcLinkToFile: - if cmpIgnoreCase(ext, splitFile(f).ext) == 0: - add(s, f) - of pcDir: walkDirRecursively(s, f, ext) - of pcLinkToDir: discard - -proc addFiles(s: var seq[string], dir, ext: string, patterns: seq[string]) = - for p in items(patterns): - if fileExists(dir / addFileExt(p, ext)): - s.add(dir / addFileExt(p, ext)) - if dirExists(dir / p): - walkDirRecursively(s, dir / p, ext) - -proc parseIniFile(c: var TConfigData) = - var - p: CfgParser - section: string # current section - var input = newFileStream(c.infile, fmRead) - if input == nil: quit("cannot open: " & c.infile) - open(p, input, c.infile) - while true: - var k = next(p) - case k.kind - of cfgEof: break - of cfgSectionStart: - section = normalize(k.section) - case section - of "project", "links", "tabs", "ticker", "documentation", "var": discard - else: echo("[Warning] Skipping unknown section: " & section) - - of cfgKeyValuePair: - var v = k.value % c.vars - c.vars[k.key] = v - - case section - of "project": - case normalize(k.key) - of "name": c.projectName = v - of "title": c.projectTitle = v - of "logo": c.logo = v - of "authors": c.authors = v - else: quit(errorStr(p, "unknown variable: " & k.key)) - of "var": discard - of "links": - let valID = v.split(';') - add(c.links, (k.key.replace('_', ' '), valID[1], valID[0])) - of "tabs": add(c.tabs, (k.key, "", v)) - of "ticker": c.ticker = v - of "documentation": - case normalize(k.key) - of "doc": addFiles(c.doc, "doc", ".rst", split(v, {';'})) - of "pdf": addFiles(c.pdf, "doc", ".rst", split(v, {';'})) - of "srcdoc": addFiles(c.srcdoc, "lib", ".nim", split(v, {';'})) - of "srcdoc2": addFiles(c.srcdoc2, "lib", ".nim", split(v, {';'})) - of "webdoc": addFiles(c.webdoc, "lib", ".nim", split(v, {';'})) - of "parallelbuild": - try: - let num = parseInt(v) - if num != 0: c.numProcessors = num - except ValueError: - quit("invalid numeric value for --parallelBuild in config") - else: quit(errorStr(p, "unknown variable: " & k.key)) - of "quotations": - let vSplit = v.split('-') - doAssert vSplit.len == 2 - c.quotations[k.key.normalize] = (vSplit[0], vSplit[1]) - else: discard - of cfgOption: quit(errorStr(p, "syntax error")) - of cfgError: quit(errorStr(p, k.msg)) - close(p) - if c.projectName.len == 0: - c.projectName = changeFileExt(extractFilename(c.infile), "") - -# ------------------- main ---------------------------------------------------- - - -proc exe(f: string): string = return addFileExt(f, ExeExt) - -proc findNim(c: TConfigData): string = - if c.nimCompiler.len > 0: return c.nimCompiler - var nim = "nim".exe - result = "bin" / nim - if fileExists(result): return - for dir in split(getEnv("PATH"), PathSep): - if fileExists(dir / nim): return dir / nim - # assume there is a symlink to the exe or something: - return nim - -proc exec(cmd: string) = - echo(cmd) - let (outp, exitCode) = osproc.execCmdEx(cmd) - if exitCode != 0: quit outp - -proc sexec(cmds: openarray[string]) = - ## Serial queue wrapper around exec. - for cmd in cmds: exec(cmd) - -proc mexec(cmds: openarray[string], processors: int) = - ## Multiprocessor version of exec - doAssert processors > 0, "nimweb needs at least one processor" - if processors == 1: - sexec(cmds) - return - let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}, - n = processors) - if r != 0: - echo "external program failed, retrying serial work queue for logs!" - sexec(cmds) - -proc buildDocSamples(c: var TConfigData, destPath: string) = - ## Special case documentation sample proc. - ## - ## TODO: consider integrating into the existing generic documentation builders - ## now that we have a single `doc` command. - exec(findNim(c) & " doc $# -o:$# $#" % - [c.nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) - -proc pathPart(d: string): string = splitFile(d).dir.replace('\\', '/') - -proc buildDoc(c: var TConfigData, destPath: string) = - # call nim for the documentation: - var - commands = newSeq[string](len(c.doc) + len(c.srcdoc) + len(c.srcdoc2)) - i = 0 - for d in items(c.doc): - commands[i] = findNim(c) & " rst2html $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(c.srcdoc): - commands[i] = findNim(c) & " doc0 $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(c.srcdoc2): - commands[i] = findNim(c) & " doc2 $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - - mexec(commands, c.numProcessors) - exec(findNim(c) & " buildIndex -o:$1/theindex.html $1" % [destPath]) - -proc buildPdfDoc(c: var TConfigData, destPath: string) = - createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - echo "pdflatex not found; no PDF documentation generated" - else: - const pdflatexcmd = "pdflatex -interaction=nonstopmode " - for d in items(c.pdf): - exec(findNim(c) & " rst2tex $# $#" % [c.nimArgs, d]) - # call LaTeX twice to get cross references right: - exec(pdflatexcmd & changeFileExt(d, "tex")) - exec(pdflatexcmd & changeFileExt(d, "tex")) - # delete all the crappy temporary files: - let pdf = splitFile(d).name & ".pdf" - let dest = destPath / pdf - removeFile(dest) - moveFile(dest=dest, source=pdf) - removeFile(changeFileExt(pdf, "aux")) - if fileExists(changeFileExt(pdf, "toc")): - removeFile(changeFileExt(pdf, "toc")) - removeFile(changeFileExt(pdf, "log")) - removeFile(changeFileExt(pdf, "out")) - removeFile(changeFileExt(d, "tex")) - -proc buildAddDoc(c: var TConfigData, destPath: string) = - # build additional documentation (without the index): - var commands = newSeq[string](c.webdoc.len) - for i, doc in pairs(c.webdoc): - commands[i] = findNim(c) & " doc2 $# --git.url:$# -o:$# $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(doc).name, "html"), doc] - mexec(commands, c.numProcessors) - -proc parseNewsTitles(inputFilename: string): seq[TRssItem] = - # Goes through each news file, returns its date/title. - result = @[] - var matches: array[3, string] - let reYearMonthDay = re(rYearMonthDay) - for kind, path in walkDir(inputFilename): - let (dir, name, ext) = path.splitFile - if ext == ".rst": - let content = readFile(path) - let title = content.splitLines()[0] - let urlPath = "news/" & name & ".html" - if content.find(reYearMonthDay, matches) >= 0: - result.add(TRssItem(year: matches[2], month: matches[1], day: matches[0], - title: title, url: "http://nim-lang.org/" & urlPath, - content: content)) - result.reverse() - -proc genUUID(text: string): string = - # Returns a valid RSS uuid, which is basically md5 with dashes and a prefix. - result = getMD5(text) - result.insert("-", 20) - result.insert("-", 16) - result.insert("-", 12) - result.insert("-", 8) - result.insert("urn:uuid:") - -proc genNewsLink(title: string): string = - # Mangles a title string into an expected news.html anchor. - result = title - result.insert("Z") - for i in 1..len(result)-1: - let letter = result[i].toLowerAscii() - if letter in validAnchorCharacters: - result[i] = letter - else: - result[i] = '-' - result.insert(rssNewsUrl & "#") - -proc generateRss(outputFilename: string, news: seq[TRssItem]) = - # Given a list of rss items generates an rss overwriting destination. - var - output: File - - if not open(output, outputFilename, mode = fmWrite): - quit("Could not write to $1 for rss generation" % [outputFilename]) - defer: output.close() - - output.write("""<?xml version="1.0" encoding="utf-8"?> -<feed xmlns="http://www.w3.org/2005/Atom"> -""") - output.write(title("Nim website news")) - output.write(link(href = rssUrl, rel = "self")) - output.write(link(href = rssNewsUrl)) - output.write(id(rssNewsUrl)) - - let now = utc(getTime()) - output.write(updatedDate($now.year, $(int(now.month) + 1), $now.monthday)) - - for rss in news: - output.write(entry( - title(xmltree.escape(rss.title)), - id(genUUID(rss.title)), - link(`type` = "text/html", rel = "alternate", - href = rss.url), - updatedDate(rss.year, rss.month, rss.day), - "<author><name>Nim</name></author>", - content(xmltree.escape(rss.content), `type` = "text") - )) - - output.write("""</feed>""") - -proc buildNewsRss(c: var TConfigData, destPath: string) = - # generates an xml feed from the web/news.rst file - let - srcFilename = "web" / "news" - destFilename = destPath / changeFileExt(splitFile(srcFilename).name, "xml") - - generateRss(destFilename, parseNewsTitles(srcFilename)) - -proc readSponsors(sponsorsFile: string): seq[Sponsor] = - result = @[] - var fileStream = newFileStream(sponsorsFile, fmRead) - if fileStream == nil: quit("Cannot open sponsors.csv file: " & sponsorsFile) - var parser: CsvParser - open(parser, fileStream, sponsorsFile) - discard readRow(parser) # Skip the header row. - while readRow(parser): - result.add(Sponsor(logo: parser.row[0], name: parser.row[1], - url: parser.row[2], thisMonth: parser.row[3].parseInt, - allTime: parser.row[4].parseInt, - since: parser.row[5], level: parser.row[6].parseInt)) - parser.close() - -proc buildSponsors(c: var TConfigData, outputDir: string) = - let sponsors = generateSponsorsPage(readSponsors(activeSponsors), - readSponsors(inactiveSponsors)) - let outFile = outputDir / "sponsors.html" - var f: File - if open(f, outFile, fmWrite): - writeLine(f, generateHtmlPage(c, "", "Our Sponsors", sponsors, "")) - close(f) - else: - quit("[Error] Cannot write file: " & outFile) - -const - cmdRst2Html = " rst2html --compileonly $1 -o:web/$2.temp web/$2.rst" - -proc buildPage(c: var TConfigData, file, title, rss: string, assetDir = "") = - exec(findNim(c) & cmdRst2Html % [c.nimArgs, file]) - var temp = "web" / changeFileExt(file, "temp") - var content: string - try: - content = readFile(temp) - except IOError: - quit("[Error] cannot open: " & temp) - var f: File - var outfile = c.webUploadOutput / "$#.html" % file - if not dirExists(outfile.splitFile.dir): - createDir(outfile.splitFile.dir) - if open(f, outfile, fmWrite): - writeLine(f, generateHTMLPage(c, file, title, content, rss, assetDir)) - close(f) - else: - quit("[Error] cannot write file: " & outfile) - removeFile(temp) - -proc buildNews(c: var TConfigData, newsDir: string, outputDir: string) = - for kind, path in walkDir(newsDir): - let (dir, name, ext) = path.splitFile - if ext == ".rst": - let title = readFile(path).splitLines()[0] - buildPage(c, tailDir(dir) / name, title, "", "../") - else: - echo("Skipping file in news directory: ", path) - -proc buildWebsite(c: var TConfigData) = - if c.ticker.len > 0: - try: - c.ticker = readFile("web" / c.ticker) - except IOError: - quit("[Error] cannot open: " & c.ticker) - for i in 0..c.tabs.len-1: - var file = c.tabs[i].val - let rss = if file in ["news", "index"]: extractFilename(rssUrl) else: "" - if '.' in file: continue - buildPage(c, file, if file == "question": "FAQ" else: file, rss) - copyDir("web/assets", c.webUploadOutput / "assets") - buildNewsRss(c, c.webUploadOutput) - buildSponsors(c, c.webUploadOutput) - buildNews(c, "web/news", c.webUploadOutput / "news") - -proc onlyDocs(c: var TConfigData) = - createDir(c.docHTMLOutput) - buildDocSamples(c, c.docHTMLOutput) - buildDoc(c, c.docHTMLOutput) - -proc main(c: var TConfigData) = - buildWebsite(c) - let docup = c.webUploadOutput / NimVersion - createDir(docup) - buildAddDoc(c, docup) - buildDocSamples(c, docup) - buildDoc(c, docup) - onlyDocs(c) - -proc json2(c: var TConfigData) = - const destPath = "web/json2" - var commands = newSeq[string](c.srcdoc2.len) - var i = 0 - for d in items(c.srcdoc2): - createDir(destPath / splitFile(d).dir) - commands[i] = findNim(c) & " jsondoc2 $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(d, "json"), d] - i.inc - - mexec(commands, c.numProcessors) - -var c: TConfigData -initConfigData(c) -parseCmdLine(c) -parseIniFile(c) -case action -of actOnlyWebsite: buildWebsite(c) -of actPdf: buildPdfDoc(c, "doc/pdf") -of actOnlyDocs: onlyDocs(c) -of actAll: main(c) -of actJson2: json2(c) diff --git a/tools/officialpackages.nim b/tools/officialpackages.nim new file mode 100644 index 000000000..633944a14 --- /dev/null +++ b/tools/officialpackages.nim @@ -0,0 +1,21 @@ +import std/[strformat, paths, dirs, envvars] +from std/os import execShellCmd + +proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + let prevPath = getEnv("PATH") + if additionalPath.len > 0: + var absolute = Path(additionalPath) + if not absolute.isAbsolute: + absolute = getCurrentDir() / absolute + echo("Adding to $PATH: ", string(absolute)) + putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & string(absolute)) + echo(cmd) + if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) + putEnv("PATH", prevPath) + +proc gitClonePackages*(names: seq[string]) = + if not dirExists(Path"pkgs"): + createDir(Path"pkgs") + for name in names: + if not dirExists(Path"pkgs" / Path(name)): + exec fmt"git clone https://github.com/nim-lang/{name} pkgs/{name}" diff --git a/tools/trimcc.nim b/tools/trimcc.nim index 959eaf310..41795e1aa 100644 --- a/tools/trimcc.nim +++ b/tools/trimcc.nim @@ -31,7 +31,7 @@ proc processIncludes(dir: string, whitelist: StringTableRef) = if ('.' notin name and "include" in path) or ("c++" in path): let n = whitelist.getOrDefault(name) if n != "processed": whitelist[name] = "found" - if name.endswith(".h"): + if name.endsWith(".h"): let n = whitelist.getOrDefault(name) if n == "found": includes(path, name, whitelist) of pcDir: processIncludes(path, whitelist) diff --git a/tools/unicode_parsedata.nim b/tools/unicode_parsedata.nim index cca377f51..bd12998d1 100644 --- a/tools/unicode_parsedata.nim +++ b/tools/unicode_parsedata.nim @@ -26,34 +26,54 @@ var proc parseData(data: seq[string]) = - for line in data: - let - fields = line.split(';') - code = fields[0].parseHexInt() - category = fields[2] - uc = fields[12] - lc = fields[13] - tc = fields[14] - + proc doAdd(firstCode, lastCode: int, category, uc, lc, tc: string) = if category notin spaces and category notin letters: - continue + return + if firstCode != lastCode: + doAssert uc == "" and lc == "" and tc == "" if uc.len > 0: - let diff = 500 + uc.parseHexInt() - code - toUpper.add (code, diff) + let diff = 500 + uc.parseHexInt() - firstCode + toUpper.add (firstCode, diff) if lc.len > 0: - let diff = 500 + lc.parseHexInt() - code - toLower.add (code, diff) + let diff = 500 + lc.parseHexInt() - firstCode + toLower.add (firstCode, diff) if tc.len > 0 and tc != uc: # if titlecase is different than uppercase - let diff = 500 + tc.parseHexInt() - code + let diff = 500 + tc.parseHexInt() - firstCode if diff != 500: - toTitle.add (code, diff) + toTitle.add (firstCode, diff) - if category in spaces: - unispaces.add code + for code in firstCode..lastCode: + if category in spaces: + unispaces.add code + else: + alphas.add code + + var idx = 0 + while idx < data.len: + let + line = data[idx] + fields = line.split(';') + code = fields[0].parseHexInt() + name = fields[1] + category = fields[2] + uc = fields[12] + lc = fields[13] + tc = fields[14] + inc(idx) + if name.endsWith(", First>"): + doAssert idx < data.len + let + nextLine = data[idx] + nextFields = nextLine.split(';') + nextCode = nextFields[0].parseHexInt() + nextName = nextFields[1] + inc(idx) + doAssert nextName.endsWith(", Last>") + doAdd(code, nextCode, category, uc, lc, tc) else: - alphas.add code + doAdd(code, code, category, uc, lc, tc) proc splitRanges(a: seq[Singlets], r: var seq[Ranges], s: var seq[Singlets]) = ## Splits `toLower`, `toUpper` and `toTitle` into separate sequences: @@ -153,18 +173,18 @@ proc createHeader(output: var string) = proc `$`(r: Ranges): string = let - start = "0x" & toHex(r.start, 5) - stop = "0x" & toHex(r.stop, 5) + start = "0x" & toHex(r.start, 5) & "'i32" + stop = "0x" & toHex(r.stop, 5) & "'i32" result = "$#, $#, $#,\n" % [start, stop, $r.diff] proc `$`(r: Singlets): string = - let code = "0x" & toHex(r.code, 5) + let code = "0x" & toHex(r.code, 5) & "'i32" result = "$#, $#,\n" % [code, $r.diff] proc `$`(r: NonLetterRanges): string = let - start = "0x" & toHex(r.start, 5) - stop = "0x" & toHex(r.stop, 5) + start = "0x" & toHex(r.start, 5) & "'i32" + stop = "0x" & toHex(r.stop, 5) & "'i32" result = "$#, $#,\n" % [start, stop] @@ -178,7 +198,7 @@ proc outputSeq(s: seq[Ranges|Singlets|NonLetterRanges], name: string, proc outputSeq(s: seq[int], name: string, output: var string) = output.add " $# = [\n" % name for i in s: - output.add " 0x$#,\n" % toHex(i, 5) + output.add " 0x$#'i32,\n" % toHex(i, 5) output.add " ]\n\n" proc outputSpaces(s: seq[int], name: string, output: var string) = diff --git a/tools/urldownloader.nim b/tools/urldownloader.nim deleted file mode 100644 index 73e4034c9..000000000 --- a/tools/urldownloader.nim +++ /dev/null @@ -1,431 +0,0 @@ - -# -# -# Windows native FTP/HTTP/HTTPS file downloader -# (c) Copyright 2017 Eugene Kabanov -# -# See the file "LICENSE", included in this -# distribution, for details about the copyright. -# - -## This module implements native Windows FTP/HTTP/HTTPS downloading feature, -## using ``urlmon.UrlDownloadToFile()``. -## -## - -when not (defined(windows) or defined(nimdoc)): - {.error: "Platform is not supported.".} - -import os - -type - DownloadOptions* = enum - ## Available download options - optUseCache, ## Use Windows cache. - optUseProgressCallback, ## Report progress via callback. - optIgnoreSecurity ## Ignore HTTPS security problems. - - DownloadStatus* = enum - ## Available download status sent to ``progress`` callback. - statusProxyDetecting, ## Automatic Proxy detection. - statusCookieSent ## Cookie will be sent with request. - statusResolving, ## Resolving URL with DNS. - statusConnecting, ## Establish connection to server. - statusRedirecting ## HTTP redirection pending. - statusRequesting, ## Sending request to server. - statusMimetypeAvailable, ## Mimetype received from server. - statusBeginDownloading, ## Download process starting. - statusDownloading, ## Download process pending. - statusEndDownloading, ## Download process finished. - statusCacheAvailable ## File found in Windows cache. - statusUnsupported ## Unsupported status. - statusError ## Error happens. - - DownloadProgressCallback* = proc(status: DownloadStatus, progress: uint, - progressMax: uint, - message: string) - ## Progress callback. - ## - ## status - ## Indicate current stage of downloading process. - ## - ## progress - ## Number of bytes currently downloaded. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## progressMax - ## Number of bytes expected to download. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## message - ## Status message, which depends on ``status`` code. - ## - ## Available messages' values: - ## - ## statusResolving - ## URL hostname to be resolved. - ## statusConnecting - ## IP address - ## statusMimetypeAvailable - ## Downloading resource MIME type. - ## statusCacheAvailable - ## Path to filename stored in Windows cache. - -type - UUID = array[4, uint32] - - LONG = clong - ULONG = culong - HRESULT = clong - DWORD = uint32 - OLECHAR = uint16 - OLESTR = ptr OLECHAR - LPWSTR = OLESTR - UINT = cuint - REFIID = ptr UUID - -const - E_NOINTERFACE = 0x80004002'i32 - E_NOTIMPL = 0x80004001'i32 - S_OK = 0x00000000'i32 - - CP_UTF8 = 65001'u32 - - IID_IUnknown = UUID([0'u32, 0'u32, 192'u32, 1174405120'u32]) - IID_IBindStatusCallback = UUID([2045430209'u32, 298760953'u32, - 2852160140'u32, 195644160'u32]) - - BINDF_GETNEWESTVERSION = 0x00000010'u32 - BINDF_IGNORESECURITYPROBLEM = 0x00000100'u32 - BINDF_RESYNCHRONIZE = 0x00000200'u32 - BINDF_NO_UI = 0x00000800'u32 - BINDF_SILENTOPERATION = 0x00001000'u32 - BINDF_PRAGMA_NO_CACHE = 0x00002000'u32 - - ERROR_FILE_NOT_FOUND = 2 - ERROR_ACCESS_DENIED = 5 - - BINDSTATUS_FINDINGRESOURCE = 1 - BINDSTATUS_CONNECTING = 2 - BINDSTATUS_REDIRECTING = 3 - BINDSTATUS_BEGINDOWNLOADDATA = 4 - BINDSTATUS_DOWNLOADINGDATA = 5 - BINDSTATUS_ENDDOWNLOADDATA = 6 - BINDSTATUS_SENDINGREQUEST = 11 - BINDSTATUS_MIMETYPEAVAILABLE = 13 - BINDSTATUS_CACHEFILENAMEAVAILABLE = 14 - BINDSTATUS_PROXYDETECTING = 32 - BINDSTATUS_COOKIE_SENT = 34 - -type - STGMEDIUM = object - tymed: DWORD - pstg: pointer - pUnkForRelease: pointer - - SECURITY_ATTRIBUTES = object - nLength*: uint32 - lpSecurityDescriptor*: pointer - bInheritHandle*: int32 - - BINDINFO = object - cbSize: ULONG - stgmedData: STGMEDIUM - szExtraInfo: LPWSTR - grfBindInfoF: DWORD - dwBindVerb: DWORD - szCustomVerb: LPWSTR - cbstgmedData: DWORD - dwOptions: DWORD - dwOptionsFlags: DWORD - dwCodePage: DWORD - securityAttributes: SECURITY_ATTRIBUTES - iid: UUID - pUnk: pointer - dwReserved: DWORD - - IBindStatusCallback = object - vtable: ptr IBindStatusCallbackVTable - options: set[DownloadOptions] - objectRefCount: ULONG - binfoFlags: DWORD - progressCallback: DownloadProgressCallback - - PIBindStatusCallback = ptr IBindStatusCallback - LPBINDSTATUSCALLBACK = PIBindStatusCallback - - IBindStatusCallbackVTable = object - QueryInterface: proc (self: PIBindStatusCallback, - riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} - AddRef: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - Release: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - OnStartBinding: proc(self: PIBindStatusCallback, - dwReserved: DWORD, pib: pointer): HRESULT - {.gcsafe, stdcall.} - GetPriority: proc(self: PIBindStatusCallback, pnPriority: ptr LONG): HRESULT - {.gcsafe, stdcall.} - OnLowResource: proc(self: PIBindStatusCallback, dwReserved: DWORD): HRESULT - {.gcsafe, stdcall.} - OnProgress: proc(self: PIBindStatusCallback, ulProgress: ULONG, - ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT - {.gcsafe, stdcall.} - OnStopBinding: proc(self: PIBindStatusCallback, hresult: HRESULT, - szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} - GetBindInfo: proc(self: PIBindStatusCallback, grfBINDF: ptr DWORD, - pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} - OnDataAvailable: proc(self: PIBindStatusCallback, grfBSCF: DWORD, - dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT - {.gcsafe, stdcall.} - OnObjectAvailable: proc(self: PIBindStatusCallback, riid: REFIID, - punk: pointer): HRESULT - {.gcsafe, stdcall.} - -template FAILED(hr: HRESULT): bool = - (hr < 0) - -proc URLDownloadToFile(pCaller: pointer, szUrl: LPWSTR, szFileName: LPWSTR, - dwReserved: DWORD, - lpfnCb: LPBINDSTATUSCALLBACK): HRESULT - {.stdcall, dynlib: "urlmon.dll", importc: "URLDownloadToFileW".} - -proc WideCharToMultiByte(CodePage: UINT, dwFlags: DWORD, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpDefaultChar: ptr char, - lpUsedDefaultChar: ptr uint32): cint - {.stdcall, dynlib: "kernel32.dll", importc: "WideCharToMultiByte".} - -proc MultiByteToWideChar(CodePage: UINT, dwFlags: DWORD, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint): cint - {.stdcall, dynlib: "kernel32.dll", importc: "MultiByteToWideChar".} -proc DeleteUrlCacheEntry(lpszUrlName: LPWSTR): int32 - {.stdcall, dynlib: "wininet.dll", importc: "DeleteUrlCacheEntryW".} - -proc `==`(a, b: UUID): bool = - result = false - if a[0] == b[0] and a[1] == b[1] and - a[2] == b[2] and a[3] == b[3]: - result = true - -proc `$`(bstr: LPWSTR): string = - var buffer: char - var count = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(buffer), 0, - nil, nil) - if count == 0: - raiseOsError(osLastError()) - else: - result = newString(count + 8) - let res = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(result[0]), count, - nil, nil) - if res == 0: - raiseOsError(osLastError()) - result.setLen(res - 1) - -proc toBstring(str: string): LPWSTR = - var buffer: OLECHAR - var count = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - addr(buffer), 0) - if count == 0: - raiseOsError(osLastError()) - else: - result = cast[LPWSTR](alloc0((count + 1) * sizeof(OLECHAR))) - let res = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - result, count) - if res == 0: - raiseOsError(osLastError()) - -proc freeBstring(bstr: LPWSTR) = - dealloc(bstr) - -proc getStatus(scode: ULONG): DownloadStatus = - case scode - of 0: result = statusError - of BINDSTATUS_PROXYDETECTING: result = statusProxyDetecting - of BINDSTATUS_REDIRECTING: result = statusRedirecting - of BINDSTATUS_COOKIE_SENT: result = statusCookieSent - of BINDSTATUS_FINDINGRESOURCE: result = statusResolving - of BINDSTATUS_CONNECTING: result = statusConnecting - of BINDSTATUS_SENDINGREQUEST: result = statusRequesting - of BINDSTATUS_MIMETYPEAVAILABLE: result = statusMimetypeAvailable - of BINDSTATUS_BEGINDOWNLOADDATA: result = statusBeginDownloading - of BINDSTATUS_DOWNLOADINGDATA: result = statusDownloading - of BINDSTATUS_ENDDOWNLOADDATA: result = statusEndDownloading - of BINDSTATUS_CACHEFILENAMEAVAILABLE: result = statusCacheAvailable - else: result = statusUnsupported - -proc addRef(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - inc(self.objectRefCount) - result = self.objectRefCount - -proc release(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - dec(self.objectRefCount) - result = self.objectRefCount - -proc queryInterface(self: PIBindStatusCallback, riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} = - pvObject[] = nil - - if riid[] == IID_IUnknown: - pvObject[] = cast[pointer](self) - elif riid[] == IID_IBindStatusCallback: - pvObject[] = cast[pointer](self) - - if not isNil(pvObject[]): - discard addRef(self) - result = S_OK - else: - result = E_NOINTERFACE - -proc onStartBinding(self: PIBindStatusCallback, dwReserved: DWORD, - pib: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc getPriority(self: PIBindStatusCallback, - pnPriority: ptr LONG): HRESULT {.gcsafe, stdcall.} = - result = E_NOTIMPL - -proc onLowResource(self: PIBindStatusCallback, - dwReserved: DWORD): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onStopBinding(self: PIBindStatusCallback, - hresult: HRESULT, szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc getBindInfo(self: PIBindStatusCallback, - grfBINDF: ptr DWORD, pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} = - var cbSize = pbindinfo.cbSize - zeroMem(cast[pointer](pbindinfo), cbSize) - pbindinfo.cbSize = cbSize - grfBINDF[] = self.binfoFlags - result = S_OK - -proc onDataAvailable(self: PIBindStatusCallback, - grfBSCF: DWORD, dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onObjectAvailable(self: PIBindStatusCallback, - riid: REFIID, punk: pointer): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc onProgress(self: PIBindStatusCallback, - ulProgress: ULONG, ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT {.gcsafe, stdcall.} = - var message: string - if optUseProgressCallback in self.options: - if not isNil(szStatusText): - message = $szStatusText - else: - message = "" - self.progressCallback(getStatus(ulStatusCode), uint(ulProgress), - uint(ulProgressMax), message) - result = S_OK - -proc newBindStatusCallback(): IBindStatusCallback = - result = IBindStatusCallback() - result.vtable = cast[ptr IBindStatusCallbackVTable]( - alloc0(sizeof(IBindStatusCallbackVTable)) - ) - result.vtable.QueryInterface = queryInterface - result.vtable.AddRef = addRef - result.vtable.Release = release - result.vtable.OnStartBinding = onStartBinding - result.vtable.GetPriority = getPriority - result.vtable.OnLowResource = onLowResource - result.vtable.OnStopBinding = onStopBinding - result.vtable.GetBindInfo = getBindInfo - result.vtable.OnDataAvailable = onDataAvailable - result.vtable.OnObjectAvailable = onObjectAvailable - result.vtable.OnProgress = onProgress - result.objectRefCount = 1 - -proc freeBindStatusCallback(v: var IBindStatusCallback) = - dealloc(v.vtable) - -proc downloadToFile*(szUrl: string, szFileName: string, - options: set[DownloadOptions] = {}, - progresscb: DownloadProgressCallback = nil) = - ## Downloads from URL specified in ``szUrl`` to local filesystem path - ## specified in ``szFileName``. - ## - ## szUrl - ## URL to download, international names are supported. - ## szFileName - ## Destination path for downloading resource. - ## options - ## Downloading options. Currently only 2 options supported. - ## progresscb - ## Callback procedure, which will be called throughout the download - ## process, indicating status and progress. - ## - ## Available downloading options: - ## - ## optUseCache - ## Try to use Windows cache when downloading. - ## optIgnoreSecurity - ## Ignore HTTPS security problems, e.g. self-signed HTTPS certificate. - ## - var bszUrl = szUrl.toBstring() - var bszFile = szFileName.toBstring() - var bstatus = newBindStatusCallback() - - bstatus.options = {} - - if optUseCache notin options: - bstatus.options.incl(optUseCache) - let res = DeleteUrlCacheEntry(bszUrl) - if res == 0: - let err = osLastError() - if err.int notin {ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}: - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(err) - - bstatus.binfoFlags = BINDF_GETNEWESTVERSION or BINDF_RESYNCHRONIZE or - BINDF_PRAGMA_NO_CACHE or BINDF_NO_UI or - BINDF_SILENTOPERATION - - if optIgnoreSecurity in options: - bstatus.binfoFlags = bstatus.binfoFlags or BINDF_IGNORESECURITYPROBLEM - - if not isNil(progresscb): - bstatus.options.incl(optUseProgressCallback) - bstatus.progressCallback = progresscb - - let res = URLDownloadToFile(nil, bszUrl, bszFile, 0, addr bstatus) - if FAILED(res): - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(OSErrorCode(res)) - - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - -when isMainModule: - proc progress(status: DownloadStatus, progress: uint, progressMax: uint, - message: string) {.gcsafe.} = - const downset: set[DownloadStatus] = {statusBeginDownloading, - statusDownloading, statusEndDownloading} - if status in downset: - var message = "Downloaded " & $progress & " of " & $progressMax & "\c" - stdout.write(message) - else: - echo "Status [" & $status & "] message = [" & $message & "]" - - downloadToFile("https://nim-lang.org/download/mingw64.7z", - "test.zip", {optUseCache}, progress) diff --git a/tools/vccexe/vccexe.nim b/tools/vccexe/vccexe.nim index 1b5b2fb82..2a43f7422 100644 --- a/tools/vccexe/vccexe.nim +++ b/tools/vccexe/vccexe.nim @@ -41,6 +41,7 @@ const platformPrefix = "--platform" sdktypePrefix = "--sdktype" sdkversionPrefix = "--sdkversion" + vctoolsetPrefix = "--vctoolset" verbosePrefix = "--verbose" vccversionSepIdx = vccversionPrefix.len @@ -49,6 +50,7 @@ const platformSepIdx = platformPrefix.len sdktypeSepIdx = sdktypePrefix.len sdkversionSepIdx = sdkversionPrefix.len + vctoolsetSepIdx = vctoolsetPrefix.len vcvarsallDefaultPath = "vcvarsall.bat" @@ -93,10 +95,12 @@ Options: --sdktype:<type> Specify the SDK flavor to use. Defaults to the Desktop SDK. <type>: {empty} | store | uwp | onecore --sdkversion:<v> Use a specific Windows SDK version: - <v> is either the full Windows 10 SDK version number or + <v> is either the full Windows 10 SDK version number or "8.1" to use the windows 8.1 SDK --verbose Echoes the command line for loading the Developer Command Prompt and the command line passed on to the secondary command. + --vctoolset Optionally specifies the Visual Studio compiler toolset to use. + By default, the environment is set to use the current Visual Studio compiler toolset. Other command line arguments are passed on to the secondary command specified by --command or to the @@ -104,12 +108,12 @@ Microsoft (R) C/C++ Optimizing Compiler if no secondary command was specified """ -proc parseVccexeCmdLine(argseq: seq[TaintedString], +proc parseVccexeCmdLine(argseq: seq[string], vccversionArg: var seq[string], printPathArg: var bool, vcvarsallArg: var string, commandArg: var string, noCommandArg: var bool, platformArg: var VccArch, sdkTypeArg: var VccPlatformType, - sdkVersionArg: var string, verboseArg: var bool, - clArgs: var seq[TaintedString]) = + sdkVersionArg: var string, vctoolsetArg: var string, verboseArg: var bool, + clArgs: var seq[string]) = ## Cannot use usual command-line argument parser here ## Since vccexe command-line arguments are intermingled ## with the secondary command-line arguments which have @@ -125,7 +129,7 @@ proc parseVccexeCmdLine(argseq: seq[TaintedString], responseargs = parseCmdLine(responsecontent) parseVccexeCmdLine(responseargs, vccversionArg, printPathArg, vcvarsallArg, commandArg, noCommandArg, platformArg, sdkTypeArg, - sdkVersionArg, verboseArg, clArgs) + sdkVersionArg, vctoolsetArg, verboseArg, clArgs) elif wargv.startsWith(vccversionPrefix): # Check for vccversion vccversionArg.add(wargv.substr(vccversionSepIdx + 1)) elif wargv.cmpIgnoreCase(printPathPrefix) == 0: # Check for printPath @@ -142,6 +146,8 @@ proc parseVccexeCmdLine(argseq: seq[TaintedString], sdkTypeArg = parseEnum[VccPlatformType](wargv.substr(sdktypeSepIdx + 1)) elif wargv.startsWith(sdkversionPrefix): # Check for sdkversion sdkVersionArg = wargv.substr(sdkversionSepIdx + 1) + elif wargv.startsWith(vctoolsetPrefix): # Check for vctoolset + vctoolsetArg = wargv.substr(vctoolsetSepIdx + 1) elif wargv.startsWith(verbosePrefix): verboseArg = true else: # Regular cl.exe argument -> store for final cl.exe invocation @@ -158,13 +164,14 @@ when isMainModule: var platformArg: VccArch var sdkTypeArg: VccPlatformType var sdkVersionArg: string + var vctoolsetArg: string var verboseArg: bool = false - var clArgs: seq[TaintedString] = @[] + var clArgs: seq[string] = @[] let wrapperArgs = commandLineParams() parseVccexeCmdLine(wrapperArgs, vccversionArg, printPathArg, vcvarsallArg, - commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, + commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg, clArgs) @@ -195,7 +202,7 @@ when isMainModule: echo "$1: $2" % [head, vcvarsallArg] # Call vcvarsall to get the appropriate VCC process environment - var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, verboseArg) + var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg) if vcvars != nil: for vccEnvKey, vccEnvVal in vcvars: putEnv(vccEnvKey, vccEnvVal) @@ -204,6 +211,11 @@ when isMainModule: if verboseArg: vccOptions.incl poEchoCmd + let currentDir = getCurrentDir() + for arg in clArgs.mitems: + if fileExists(arg): + arg = relativePath(arg, currentDir) + # Default to the cl.exe command if no secondary command was specified if commandArg.len < 1: commandArg = "cl.exe" diff --git a/tools/vccexe/vcvarsall.nim b/tools/vccexe/vcvarsall.nim index 29d13cc7e..73b103e3c 100644 --- a/tools/vccexe/vcvarsall.nim +++ b/tools/vccexe/vcvarsall.nim @@ -33,7 +33,7 @@ type vccplatUWP = "uwp", ## Universal Windows Platform (UWP) Application vccplatOneCore = "onecore" # Undocumented platform type in the Windows SDK, probably XBox One SDK platform type. -proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = "", verbose: bool = false): StringTableRef = +proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version, vctoolset: string = "", verbose: bool = false): StringTableRef = ## Returns a string table containing the proper process environment to successfully execute VCC compile commands for the specified SDK version, CPU architecture and platform type. ## ## path @@ -44,6 +44,8 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type ## The compile target Platform Type. Defaults to the Windows Desktop platform, i.e. a regular Windows executable binary. ## sdk_version ## The Windows SDK version to use. + ## vctoolset + ## Visual Studio compiler toolset to use. ## verbose ## Echo the command-line passed on to the system to load the VCC environment. Defaults to `false`. @@ -63,6 +65,9 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type if sdk_version.len > 0: args.add(sdk_version) + + if vctoolset.len > 0: + args.add("-vcvars_ver="&vctoolset) let argStr = args.join " " diff --git a/tools/website.nimf b/tools/website.nimf deleted file mode 100644 index bde84b32f..000000000 --- a/tools/website.nimf +++ /dev/null @@ -1,266 +0,0 @@ -#? stdtmpl | standard -#proc generateHTMLPage(c: var TConfigData, currentTab, title, content, rss, -# rootDir = ""): string = -# result = "" -<!DOCTYPE html> -<html> - <head> - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <title>${title} - $c.projectTitle</title> - <link rel="stylesheet" type="text/css" href="${rootDir}assets/style.css?t=2320" /> - <link rel="shortcut icon" href="${rootDir}assets/images/favicon.ico"> - #if len(rss) > 0: - <link href="$rss" title="Recent changes" type="application/atom+xml" rel="alternate"> - #end if - </head> - <body> - <div id="bountysource"> - <a href="https://salt.bountysource.com/teams/nim"> - <div class="page-layout" style="padding: 2pt 2pt 2pt 30pt"> - <img src="${rootDir}assets/bountysource/bountysource.png" style="width: 20px; float: left;"> - <span style="margin-left: 10pt; float: left; margin-top: 2pt;">Fund Nim and help us develop it further!</span> - <img src="https://api.bountysource.com/badge/team?team_id=19072&style=raised" style="margin-top: 2pt; margin-left: 10pt"/> - </div> - </a> - </div> - - <header id="head"> - <div class="page-layout tall"> - <div id="head-logo"></div> - <a id="head-logo-link" href="${rootDir}index.html"></a> - <nav id="head-links"> - #for i in 0.. c.tabs.len-1: - # let t = c.tabs[i].val - # if t != "index" and t != "community" and t != "news": - # let name = c.tabs[i].key - # if currentTab == t: - <a class="active" - # else: - <a - # end if - # if t.contains('.'): - href="${t}" title = "$c.projectName - $name">$name</a> - # else: - href="${rootDir}${t}.html" title = "$c.projectName - $name">$name</a> - # end if - # end if - #end for - </nav> - </div> - </header> - -# if currentTab == "index": - <section id="neck" class="home"> -# else: - <section id="neck"> -# end - <div class="page-layout tall"> - <div id="glow-arrow"></div> - -# if currentTab == "index": - <div id="slideshow"> - <!-- slides --> - <div id="slide0" class="active codeslide2"> - <div> - <h2>Nim is simple..</h2> -<pre> -<span class="cmt"># compute average line length</span> -<span class="kwd">var</span> -<span class="tab"> </span>sum = <span class="val">0</span> -<span class="tab end"> </span>count = <span class="val">0</span> - -<span class="kwd">for</span> line <span class="kwd">in</span> stdin.lines: -<span class="tab"> </span>sum += line.len -<span class="tab end"> </span>count += <span class="val">1</span> - -echo(<span class="val">"Average line length: "</span>, - <span class="kwd">if</span> count > <span class="val">0</span>: sum / count <span class="kwd">else</span>: <span class="val">0</span>) -</pre> - </div> - <div> - <h2>..and type safe...</h2> -<pre> -<span class="cmt"># create and greet someone</span> -<span class="kwd">type</span> <span class="def">Person</span> = <span class="kwd">object</span> -<span class="tab"> </span>name: <span class="typ">string</span> -<span class="tab end"> </span>age: <span class="typ">int</span> - -<span class="kwd">proc</span> <span class="def">greet</span>(p: <span class="typ">Person</span>) = -<span class="tab"> </span>echo <span class="val">"Hi, I'm "</span>, p.name, <span class="val">"."</span> -<span class="tab end"> </span>echo <span class="val">"I am "</span>, p.age, <span class="val">" years old."</span> - -<span class="kwd">let</span> p = <span class="typ">Person</span>(name:<span class="val">"Jon"</span>, age:<span class="val">18</span>) -p.greet() <span class="cmt"># or greet(p)</span> -</pre> - </div> - </div> - <div id="slide1" class="codeslide3"> - <div> - <h2>C FFI is easy in Nim..</h2> -<pre> -<span class="cmt"># declare a C procedure..</span> -<span class="kwd">proc</span> <span class="def">unsafeScanf</span>(f: <span class="typ">File</span>, s: <span class="typ">cstring</span>) -<span class="tab"> </span>{.varargs, -<span class="tab"> </span>importc: <span class="val">"fscanf"</span>, -<span class="tab end"> </span>header: <span class="val">"<stdio.h>"</span>.} - -<span class="cmt"># ..and use it...</span> -<span class="kwd">var</span> x: <span class="typ">cint</span> -stdin.unsafeScanf(<span class="val">"%d"</span>, <span class="kwd">addr</span> x) -</pre> - <p><span class="desc"><b>Compile and run with:</b><br> $ nim c -r example.nim</span></p> - </div> - <div> - <h2>..and DSLs are too...</h2> -<pre> -<span class="cmt"># a simple html server</span> -<span class="kwd">import</span> - jester, asyncdispatch, htmlgen - -<span class="kwd">routes</span>: -<span class="tab"> </span><span class="kwd">get</span> <span class="val">"/"</span>: -<span class="tab end"> <span class="tab end"> </span></span><span class="kwd">resp</span> h1(<span class="val">"Hello world"</span>) - -runForever() -</pre> - <p><span class="desc"><b>View in browser at:</b><br> localhost:5000</span></p> - </div> - </div> - <div id="slide2" class="niaslide"> - <a href="news/e030_nim_in_action_in_production.html"> - <img src="${rootDir}assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!"/> - </a> - </div> - <div id="slide3" class="niaslide"> - <a href="sponsors.html"> - <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/> - </a> - </div> - </div> - <div id="slideshow-nav"> - <div id="slideControl0" onclick="slideshow_click(0)" class="active"></div> - <div id="slideControl1" onclick="slideshow_click(1)"></div> - <div id="slideControl2" onclick="slideshow_click(2)"></div> - <div id="slideControl3" onclick="slideshow_click(3)"></div> - </div> -# end - <aside id="sidebar"> - -# if len(c.links) > 0: - <h3>More Links</h3> - <div id="sidebar-links"> -# for i in 0..c.links.len-1: - <a href="${c.links[i].val}" id="${c.links[i].id}">${c.links[i].key}</a> -# end for - </div> -# end if -# if len(c.ticker) > 0: - <h3 class="blue">Latest News</h3> - <div id="sidebar-news"> - ${c.ticker % rootDir} - </div> -# end if - </aside> - </div> - </section> - - <section id="body"> - <div id="body-border"></div> - <div id="glow-line"></div> - <div class="page-layout"> - <article id="content" class="page"> - $content - </article> - </div> - </section> - - <!--- #foot ---> - <footer id="foot" class="home"> - <div class="page-layout tall"> - <div id="foot-links"> - <div> - <h4>Documentation</h4> - <a href="${rootDir}documentation.html">Stable Documentation</a> - <a href="${rootDir}learn.html">Learning Resources</a> - <!-- <a href="">Development Documentation</a> --> - <a href="https://github.com/nim-lang/nim">Issues & Requests</a> - </div> - <div> - <h4>Community</h4> - <a href="https://forum.nim-lang.org">User Forum</a> - <a href="http://webchat.freenode.net/?channels=nim">Online IRC</a> - <a href="https://irclogs.nim-lang.org/">IRC Logs</a> - </div> - </div> - <div id="foot-legal"> - <h4>Written in Nim - Powered by <a href="https://github.com/dom96/jester">Jester</a></h4> - Web Design by <a href="http://reign-studios.net/philipwitte/">Philip Witte</a> & <a href="http://picheta.me/">Dominik Picheta</a><br> - Copyright © 2017 - <a href="https://nim-lang.org/blog/">Andreas Rumpf</a> & <a href="https://github.com/nim-lang/nim/graphs/contributors">Contributors</a> - </div> - </div> - </footer> - -# if currentTab == "index": - <script src="${rootDir}assets/index.js"></script> -# end if -# if c.gaId.len != 0: - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', '${c.gaId}', 'nim-lang.org'); - ga('send', 'pageview'); - </script> -# end if -</body> -</html> -#end proc -# -# -#proc generateSponsors(sponsors: seq[Sponsor]): string = -#result = "" -#for sponsor in sponsors: - <dt class="level-${sponsor.level}"> - #if sponsor.url.len > 0: - <a href="${sponsor.url}" target="_blank">${sponsor.name}</a> - #else: - ${sponsor.name} - #end if - </dt> - <dd class="logo"> - #if sponsor.logo.len > 0: - <a href="${sponsor.url}" target="_blank"> - <img alt="${sponsor.name}'s logo" src="${sponsor.logo}"/> - </a> - #end if - </dd> - <dd class="this_month"> - Donated <b>$$${sponsor.thisMonth}</b> this month - </dd> - <dd class="legend"> - Donated $$${sponsor.allTime} in total since ${sponsor.since} - </dd> -#end for -#end proc -#proc generateSponsorsPage(activeSponsors, inactiveSponsors: seq[Sponsor]): string = -#result = "" -<h1 id="our-current-sponsors">Our Current Sponsors</h1> -<p>This section lists the companies and individuals that are, very kindly, contributing a -monthly amount to help sustain Nim's development. For more details take a -look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> -<p class="lastUpdate">Last updated: ${getTime().utc().format("dd/MM/yyyy")}</p> -<dl> -${generateSponsors(activeSponsors)} -</dl> -# -<h1 id="our-past-sponsors">Our Past Sponsors</h1> -<p>This section lists the companies and individuals that have contributed -money in the past to help sustain Nim's development. For more details take a -look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> -<dl> -${generateSponsors(inactiveSponsors)} -</dl> -# -#end proc |