diff options
Diffstat (limited to 'tools')
38 files changed, 2779 insertions, 2338 deletions
diff --git a/tools/atlas/parse_requires.nim b/tools/atlas/parse_requires.nim new file mode 100644 index 000000000..66879d04f --- /dev/null +++ b/tools/atlas/parse_requires.nim @@ -0,0 +1,101 @@ +## Utility API for Nim package managers. +## (c) 2021 Andreas Rumpf + +import std / strutils +import ".." / ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils] + +type + NimbleFileInfo* = object + requires*: seq[string] + srcDir*: string + tasks*: seq[(string, string)] + +proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = + case n.kind + of nkStmtList, nkStmtListExpr: + for child in n: + extract(child, conf, result) + of nkCallKinds: + if n[0].kind == nkIdent: + case n[0].ident.s + of "requires": + for i in 1..<n.len: + var ch = n[i] + while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0: ch = ch.lastSon + if ch.kind in {nkStrLit..nkTripleStrLit}: + result.requires.add ch.strVal + else: + localError(conf, ch.info, "'requires' takes string literals") + of "task": + if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}: + result.tasks.add((n[1].ident.s, n[2].strVal)) + else: discard + of nkAsgn, nkFastAsgn: + if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0: + if n[1].kind in {nkStrLit..nkTripleStrLit}: + result.srcDir = n[1].strVal + else: + localError(conf, n[1].info, "assignments to 'srcDir' must be string literals") + else: + discard + +proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo = + ## Extract the `requires` information from a Nimble file. This does **not** + ## evaluate the Nimble file. Errors are produced on stderr/stdout and are + ## formatted as the Nim compiler does it. The parser uses the Nim compiler + ## as an API. The result can be empty, this is not an error, only parsing + ## errors are reported. + var conf = newConfigRef() + conf.foreignPackageNotes = {} + conf.notes = {} + conf.mainPackageNotes = {} + + let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + extract(parseAll(parser), conf, result) + closeParser(parser) + +const Operators* = {'<', '>', '=', '&', '@', '!', '^'} + +proc token(s: string; idx: int; lit: var string): int = + var i = idx + if i >= s.len: return i + while s[i] in Whitespace: inc(i) + case s[i] + of Letters, '#': + lit.add s[i] + inc i + while i < s.len and s[i] notin (Whitespace + {'@', '#'}): + lit.add s[i] + inc i + of '0'..'9': + while i < s.len and s[i] in {'0'..'9', '.'}: + lit.add s[i] + inc i + of '"': + inc i + while i < s.len and s[i] != '"': + lit.add s[i] + inc i + inc i + of Operators: + while i < s.len and s[i] in Operators: + lit.add s[i] + inc i + else: + lit.add s[i] + inc i + result = i + +iterator tokenizeRequires*(s: string): string = + var start = 0 + var tok = "" + while start < s.len: + tok.setLen 0 + start = token(s, start, tok) + yield tok + +when isMainModule: + for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"): + echo x diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index d13f28c33..46bc39470 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -1,6 +1,6 @@ ##[ avoid code duplication in CI pipelines. -For now, this is only used for openbsd, but there is a lot of other code +For now, this is only used for openbsd + freebsd, but there is a lot of other code duplication that could be removed. ## usage @@ -10,22 +10,15 @@ nim r tools/ci_generate.nim ``` ]## -import std/strformat +import std/[strformat, os] -proc genCIopenbsd(batch: int, num: int): string = +const doNotEdit = "DO NO EDIT DIRECTLY! auto-generated by `nim r tools/ci_generate.nim`" +proc genCiBsd(header: string, batch: int, num: int): string = result = fmt""" -## do not edit directly; auto-generated by `nim r tools/ci_generate.nim` +## {doNotEdit} + +{header} -image: openbsd/latest -packages: -- gmake -- sqlite3 -- node -- boehm-gc -- pcre -- sfml -- sdl2 -- libffi sources: - https://github.com/nim-lang/Nim environment: @@ -33,32 +26,116 @@ environment: CC: /usr/bin/clang tasks: - setup: | + set -e cd Nim - git clone --depth 1 -q https://github.com/nim-lang/csources.git - gmake -C csources -j $(sysctl -n hw.ncpuonline) - bin/nim c koch + . ci/funs.sh && nimBuildCsourcesIfNeeded echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv - test: | + set -e cd Nim - if ! ./koch runCI; then - nim c -r tools/ci_testresults.nim - exit 1 - fi + . ci/funs.sh && nimInternalBuildKochAndRunCI triggers: - action: email condition: failure to: Andreas Rumpf <rumpf_a@web.de> """ +proc genBuildExtras(echoRun, koch, nim: string): string = + result = fmt""" +{echoRun} {nim} c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch +{echoRun} {koch} boot -d:release --skipUserCfg --skipParentCfg --hints:off +{echoRun} {koch} tools --skipUserCfg --skipParentCfg --hints:off +""" + +proc genWindowsScript(buildAll: bool): string = + result = fmt""" +@echo off +rem {doNotEdit} +rem Build development version of the compiler; can be rerun safely +rem bare bones version of ci/funs.sh adapted for windows. + +rem Read in some common shared variables (shared with other tools), +rem see https://stackoverflow.com/questions/3068929/how-to-read-file-contents-into-a-variable-in-a-batch-file +for /f "delims== tokens=1,2" %%G in (config/build_config.txt) do set %%G=%%H +SET nim_csources=bin\nim_csources_%nim_csourcesHash%.exe +echo "building from csources: %nim_csources%" + +if not exist %nim_csourcesDir% ( + git clone -q --depth 1 -b %nim_csourcesBranch% %nim_csourcesUrl% %nim_csourcesDir% +) + +if not exist %nim_csources% ( + cd %nim_csourcesDir% + git checkout %nim_csourcesHash% + echo "%PROCESSOR_ARCHITECTURE%" + if "%PROCESSOR_ARCHITECTURE%"=="AMD64" ( + SET ARCH=64 + ) + CALL build.bat + cd .. + copy /y bin\nim.exe %nim_csources% +) +""" + + if buildAll: + result.add genBuildExtras("", "koch", r"bin\nim.exe") + +proc genPosixScript(): string = + result = fmt""" +#!/bin/sh +# {doNotEdit} + +# build development version of the compiler; can be rerun safely. +# arguments can be passed, e.g.: +# CC=gcc ucpu=amd64 uos=darwin + +set -u # error on undefined variables +set -e # exit on first error + +. ci/funs.sh +nimBuildCsourcesIfNeeded "$@" + +{genBuildExtras("echo_run", "./koch", "bin/nim")} +""" + proc main()= + let dir = ".builds" # not too large to be resource friendly, refs bug #17107 let num = 2 # if you reduce this, make sure to remove files that shouldn't be generated, # or better, do the cleanup logic here e.g.: `rm .builds/openbsd_*` + let headerFreebsd = """ +# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd +image: freebsd/latest +packages: +- databases/sqlite3 +- devel/boehm-gc-threaded +- devel/pcre +- devel/sdl20 +- devel/sfml +- www/node +- devel/gmake +""" + + let headerOpenbsd = """ +image: openbsd/latest +packages: +- gmake +- sqlite3 +- node +- boehm-gc +- pcre +- sfml +- sdl2 +- libffi +""" + for i in 0..<num: - let file = fmt".builds/openbsd_{i}.yml" - let code = genCIopenbsd(i, num) - writeFile(file, code) + writeFile(dir / fmt"openbsd_{i}.yml", genCiBsd(headerOpenbsd, i, num)) + writeFile(dir / "freebsd.yml", genCiBsd(headerFreebsd, 0, 1)) + writeFile("build_all.sh", genPosixScript()) + writeFile("build_all.bat", genWindowsScript(buildAll = true)) + writeFile("ci/build_autogen.bat", genWindowsScript(buildAll = false)) when isMainModule: main() diff --git a/tools/compiler.gdb b/tools/compiler.gdb new file mode 100644 index 000000000..c81f47152 --- /dev/null +++ b/tools/compiler.gdb @@ -0,0 +1,39 @@ +# create a breakpoint on `debugutils.enteringDebugSection` +define enable_enteringDebugSection + break -function enteringDebugSection + # run these commands once breakpoint enteringDebugSection is hit + command + # enable all breakpoints and watchpoints + enable + # continue execution + cont + end +end + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +define enable_exitingDebugSection + break -function exitingDebugSection + # run these commands once breakpoint exitingDebugSection is hit + command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # continue execution + cont + end +end + +# some commands can't be set until the process is running, so set an entry breakpoint +break -function NimMain +# run these commands once breakpoint NimMain is hit +command + # disable all breakpoints and watchpoints + disable + # but enable the enteringDebugSection breakpoint + enable_enteringDebugSection + # no longer need this breakpoint + delete -function NimMain + # continue execution + cont +end diff --git a/tools/compiler.lldb b/tools/compiler.lldb new file mode 100644 index 000000000..e0b375055 --- /dev/null +++ b/tools/compiler.lldb @@ -0,0 +1,40 @@ +# create a breakpoint on `debugutils.enteringDebugSection` named enteringDebugSection +breakpoint set -n 'enteringDebugSection' -N enteringDebugSection +# run these commands once breakpoint enteringDebugSection is hit +breakpoint command add enteringDebugSection + # enable all breakpoints + breakpoint enable + # enable all watchpoints + # watchpoint enable # FIXME: not currently working for unknown reason + # continue execution + continue +DONE + +# create a breakpoint on `debugutils.exitingDebugSection` named exitingDebugSection +breakpoint set -n 'exitingDebugSection' -N exitingDebugSection +# run these commands once breakpoint exitingDebugSection is hit +breakpoint command add exitingDebugSection + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + breakpoint enable enteringDebugSection + # continue execution + continue +DONE + +# some commands can't be set until the process is running, so set an entry breakpoint +breakpoint set -n NimMain -N NimMain +# run these commands once breakpoint NimMain is hit +breakpoint command add NimMain + # disable all breakpoints + breakpoint disable + # disable all watchpoints + # watchpoint disable # FIXME: not currently working for unknown reason + # enable the enteringDebugSection breakpoint though + breakpoint enable enteringDebugSection + # no longer need this breakpoint + breakpoint delete NimMain + # continue execution + continue +DONE diff --git a/tools/debug/customdebugtype.nim b/tools/debug/customdebugtype.nim new file mode 100644 index 000000000..f48979661 --- /dev/null +++ b/tools/debug/customdebugtype.nim @@ -0,0 +1,72 @@ +## This is a demo file containing an example of how to +## create custom LLDB summaries and objects with synthetic +## children. These are implemented in Nim and called from the Python +## nimlldb.py module. +## +## For summaries, prefix your proc names with "lldbDebugSummary", use +## the `{.exportc.}` pragma, and return a string. Also, any `$` proc +## that is available will be used for a given type. +## +## For creating a synthetic object (LLDB will display the children), use +## the prefix "lldbDebugSynthetic", use the `{.exportc.}` pragma, and +## return any Nim object, array, or sequence. Returning a Nim object +## will display the fields and values of the object as children. +## Returning an array or sequence will display children with the index +## surrounded by square brackets as the key name +## +## You may also return a Nim table that contains the string +## "LLDBDynamicObject" (case insensitive). This allows for dynamic +## fields to be created at runtime instead of at compile time if you +## return a Nim object as mentioned above. See the proc +## `lldbDebugSyntheticDynamicFields` below for an example + +import intsets +import tables + +type + CustomType* = object of RootObj # RootObj is not necessary, but can be used + myField*: int + + DynamicFields* = object + customField*: string + + CustomSyntheticReturn* = object + differentField*: float + + LLDBDynamicObject = object + fields: TableRef[string, int] + + LLDBDynamicObjectDynamicFields = object + fields: TableRef[string, string] + +proc lldbDebugSummaryCustomType*(ty: CustomType): string {.exportc.} = + ## Will display "CustomType(myField: <int_val>)" as a summary + result = "CustomType" & $ty + +proc lldbDebugSyntheticCustomType*(ty: CustomType): CustomSyntheticReturn {.exportc.} = + ## Will display differentField: <float_val> as a child of CustomType instead of + ## myField: <int_val> + result = CustomSyntheticReturn(differentField: ty.myField.float) + +proc lldbDebugSyntheticDynamicFields*(ty: DynamicFields): LLDBDynamicObjectDynamicFields {.exportc.} = + ## Returning an object that contains "LLDBDynamicObject" in the type name will expect an + ## object with one property that is a Nim Table/TableRef. If the key is a string, + ## it will appear in the debugger like an object field name. The value will be whatever you + ## set it to here as well. + let fields = {"customFieldName": ty.customField & " MORE TEXT"}.newTable() + return LLDBDynamicObjectDynamicFields(fields: fields) + +proc lldbDebugSummaryIntSet*(intset: IntSet): string {.exportc.} = + ## This will print the object in the LLDB summary just as Nim prints it + result = $intset + +proc lldbDebugSyntheticIntSet*(intset: IntSet): seq[int] {.exportc.} = + ## This will create a synthetic object to make it so that IntSet + ## will appear as a Nim object in the LLDB debugger window + ## + ## returning a seq here will display children like: + ## [0]: <child_value> + ## + result = newSeqOfCap[int](intset.len) + for val in intset: + result.add(val) diff --git a/tools/nim-gdb.py b/tools/debug/nim-gdb.py index 8143b94d5..8c9854bda 100644 --- a/tools/nim-gdb.py +++ b/tools/debug/nim-gdb.py @@ -1,6 +1,7 @@ import gdb import re import sys +import traceback # some feedback that the nim runtime support is loading, isn't a bad # thing at all. @@ -13,14 +14,23 @@ def printErrorOnce(id, message): global errorSet if id not in errorSet: errorSet.add(id) - gdb.write(message, gdb.STDERR) + gdb.write("printErrorOnce: " + message, gdb.STDERR) +def debugPrint(x): + gdb.write(str(x) + "\n", gdb.STDERR) + +NIM_STRING_TYPES = ["NimStringDesc", "NimStringV2"] ################################################################################ ##### Type pretty printers ################################################################################ -type_hash_regex = re.compile("^\w*_([A-Za-z0-9]*)$") +type_hash_regex = re.compile("^([A-Za-z0-9]*)_([A-Za-z0-9]*)_+([A-Za-z0-9]*)$") + +def getNimName(typ): + if m := type_hash_regex.match(typ): + return m.group(2) + return f"unknown <{typ}>" def getNimRti(type_name): """ Return a ``gdb.Value`` object for the Nim Runtime Information of ``type_name``. """ @@ -29,12 +39,19 @@ def getNimRti(type_name): # every non trivial Nim type. m = type_hash_regex.match(type_name) if m: - try: - return gdb.parse_and_eval("NTI__" + m.group(1) + "_") - except: - return None - -def getNameFromNimRti(rti_val): + lookups = [ + "NTI" + m.group(2).lower() + "__" + m.group(3) + "_", + "NTI" + "__" + m.group(3) + "_", + "NTI" + m.group(2).replace("colon", "58").lower() + "__" + m.group(3) + "_" + ] + for l in lookups: + try: + return gdb.parse_and_eval(l) + except: + pass + None + +def getNameFromNimRti(rti): """ Return name (or None) given a Nim RTI ``gdb.Value`` """ try: # sometimes there isn't a name field -- example enums @@ -60,7 +77,7 @@ class NimTypeRecognizer: 'NIM_BOOL': 'bool', - 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string' + 'NIM_CHAR': 'char', 'NCSTRING': 'cstring', 'NimStringDesc': 'string', 'NimStringV2': 'string' } # object_type_pattern = re.compile("^(\w*):ObjectType$") @@ -95,6 +112,12 @@ class NimTypeRecognizer: result = self.type_map_static.get(tname, None) if result: return result + elif tname.startswith("tyEnum_"): + return getNimName(tname) + elif tname.startswith("tyTuple__"): + # We make the name be the field types (Just like in Nim) + fields = ", ".join([self.recognize(field.type) for field in type_obj.fields()]) + return f"({fields})" rti = getNimRti(tname) if rti: @@ -128,7 +151,7 @@ class DollarPrintFunction (gdb.Function): "Nim's equivalent of $ operator as a gdb function, available in expressions `print $dollar(myvalue)" dollar_functions = re.findall( - 'NimStringDesc \*(dollar__[A-z0-9_]+?)\(([^,)]*)\);', + r'(?:NimStringDesc \*|NimStringV2)\s?(dollar__[A-z0-9_]+?)\(([^,)]*)\);', gdb.execute("info functions dollar__", True, True) ) @@ -137,25 +160,23 @@ class DollarPrintFunction (gdb.Function): @staticmethod - def invoke_static(arg): - - if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name == "NimStringDesc": + def invoke_static(arg, ignore_errors = False): + if arg.type.code == gdb.TYPE_CODE_PTR and arg.type.target().name in NIM_STRING_TYPES: return arg - argTypeName = str(arg.type) - for func, arg_typ in DollarPrintFunction.dollar_functions: # this way of overload resolution cannot deal with type aliases, # therefore it won't find all overloads. if arg_typ == argTypeName: - func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() + func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTION_DOMAIN).value() return func_value(arg) elif arg_typ == argTypeName + " *": - func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTIONS_DOMAIN).value() + func_value = gdb.lookup_global_symbol(func, gdb.SYMBOL_FUNCTION_DOMAIN).value() return func_value(arg.address) - printErrorOnce(argTypeName, "No suitable Nim $ operator found for type: " + argTypeName + "\n") + if not ignore_errors: + debugPrint(f"No suitable Nim $ operator found for type: {getNimName(argTypeName)}\n") return None def invoke(self, arg): @@ -176,11 +197,11 @@ class NimStringEqFunction (gdb.Function): @staticmethod def invoke_static(arg1,arg2): - if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name == "NimStringDesc": + if arg1.type.code == gdb.TYPE_CODE_PTR and arg1.type.target().name in NIM_STRING_TYPES: str1 = NimStringPrinter(arg1).to_string() else: str1 = arg1.string() - if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name == "NimStringDesc": + if arg2.type.code == gdb.TYPE_CODE_PTR and arg2.type.target().name in NIM_STRING_TYPES: str2 = NimStringPrinter(arg1).to_string() else: str2 = arg2.string() @@ -192,6 +213,7 @@ class NimStringEqFunction (gdb.Function): NimStringEqFunction() + ################################################################################ ##### GDB Command, equivalent of Nim's $ operator ################################################################################ @@ -207,7 +229,7 @@ class DollarPrintCmd (gdb.Command): strValue = DollarPrintFunction.invoke_static(param) if strValue: gdb.write( - NimStringPrinter(strValue).to_string() + "\n", + str(NimStringPrinter(strValue)) + "\n", gdb.STDOUT ) @@ -245,7 +267,6 @@ class KochCmd (gdb.Command): os.path.dirname(os.path.dirname(__file__)), "koch") def invoke(self, argument, from_tty): - import os subprocess.run([self.binary] + gdb.string_to_argv(argument)) KochCmd() @@ -299,8 +320,14 @@ class NimBoolPrinter: ################################################################################ +def strFromLazy(strVal): + if isinstance(strVal, str): + return strVal + else: + return strVal.value().string("utf-8") + class NimStringPrinter: - pattern = re.compile(r'^NimStringDesc \*$') + pattern = re.compile(r'^(NimStringDesc \*|NimStringV2)$') def __init__(self, val): self.val = val @@ -310,26 +337,18 @@ class NimStringPrinter: def to_string(self): if self.val: - l = int(self.val['Sup']['len']) - return self.val['data'].lazy_string(encoding="utf-8", length=l) + if self.val.type.name == "NimStringV2": + l = int(self.val["len"]) + data = self.val["p"]["data"] + else: + l = int(self.val['Sup']['len']) + data = self.val["data"] + return data.lazy_string(encoding="utf-8", length=l) else: return "" -# class NimStringPrinter: -# pattern = re.compile(r'^NimStringDesc$') - -# def __init__(self, val): -# self.val = val - -# def display_hint(self): -# return 'string' - -# def to_string(self): -# if self.val: -# l = int(self.val['Sup']['len']) -# return self.val['data'].lazy_string(encoding="utf-8", length=l) -# else: -# return "" + def __str__(self): + return strFromLazy(self.to_string()) class NimRopePrinter: pattern = re.compile(r'^tyObject_RopeObj__([A-Za-z0-9]*) \*$') @@ -352,59 +371,43 @@ class NimRopePrinter: ################################################################################ -# proc reprEnum(e: int, typ: PNimType): string {.compilerRtl.} = -# ## Return string representation for enumeration values -# var n = typ.node -# if ntfEnumHole notin typ.flags: -# let o = e - n.sons[0].offset -# if o >= 0 and o <% typ.node.len: -# return $n.sons[o].name -# else: -# # ugh we need a slow linear search: -# var s = n.sons -# for i in 0 .. n.len-1: -# if s[i].offset == e: -# return $s[i].name -# result = $e & " (invalid data!)" - def reprEnum(e, typ): - """ this is a port of the nim runtime function `reprEnum` to python """ + # Casts the value to the enum type and then calls the enum printer e = int(e) - n = typ["node"] - flags = int(typ["flags"]) - # 1 << 2 is {ntfEnumHole} - if ((1 << 2) & flags) == 0: - o = e - int(n["sons"][0]["offset"]) - if o >= 0 and 0 < int(n["len"]): - return n["sons"][o]["name"].string("utf-8", "ignore") - else: - # ugh we need a slow linear search: - s = n["sons"] - for i in range(0, int(n["len"])): - if int(s[i]["offset"]) == e: - return s[i]["name"].string("utf-8", "ignore") + val = gdb.Value(e).cast(typ) + return strFromLazy(NimEnumPrinter(val).to_string()) - return str(e) + " (invalid data!)" +def enumNti(typeNimName, idString): + typeInfoName = "NTI" + typeNimName.lower() + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + if nti is None: + typeInfoName = "NTI" + "__" + idString + "_" + nti = gdb.lookup_global_symbol(typeInfoName) + return (typeInfoName, nti) class NimEnumPrinter: - pattern = re.compile(r'^tyEnum_(\w*)__([A-Za-z0-9]*)$') + pattern = re.compile(r'^tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') + enumReprProc = gdb.lookup_global_symbol("reprEnum", gdb.SYMBOL_FUNCTION_DOMAIN) def __init__(self, val): self.val = val typeName = self.val.type.name match = self.pattern.match(typeName) self.typeNimName = match.group(1) - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) - - if self.nti is None: - printErrorOnce(typeInfoName, f"NimEnumPrinter: lookup global symbol '{typeInfoName}' failed for {typeName}.\n") + typeInfoName, self.nti = enumNti(self.typeNimName, match.group(2)) def to_string(self): - if self.nti: - arg0 = self.val - arg1 = self.nti.value(gdb.newest_frame()) - return reprEnum(arg0, arg1) + if NimEnumPrinter.enumReprProc and self.nti: + # Use the old runtimes enumRepr function. + # We call the Nim proc itself so that the implementation is correct + f = gdb.newest_frame() + # We need to strip the quotes so it looks like an enum instead of a string + reprProc = NimEnumPrinter.enumReprProc.value() + return str(reprProc(self.val, self.nti.value(f).address)).strip('"') + elif dollarResult := DollarPrintFunction.invoke_static(self.val): + # New runtime doesn't use enumRepr so we instead try and call the + # dollar function for it + return str(NimStringPrinter(dollarResult)) else: return self.typeNimName + "(" + str(int(self.val)) + ")" @@ -414,34 +417,27 @@ class NimSetPrinter: ## the set printer is limited to sets that fit in an integer. Other ## sets are compiled to `NU8 *` (ptr uint8) and are invisible to ## gdb (currently). - pattern = re.compile(r'^tySet_tyEnum_(\w*)_([A-Za-z0-9]*)$') + pattern = re.compile(r'^tySet_tyEnum_([A-Za-z0-9]+)__([A-Za-z0-9]*)$') def __init__(self, val): self.val = val - match = self.pattern.match(self.val.type.name) - self.typeNimName = match.group(1) - - typeInfoName = "NTI__" + match.group(2) + "_" - self.nti = gdb.lookup_global_symbol(typeInfoName) - - if self.nti is None: - printErrorOnce(typeInfoName, "NimSetPrinter: lookup global symbol '"+ typeInfoName +" failed for " + self.val.type.name + ".\n") + typeName = self.val.type.name + match = self.pattern.match(typeName) + self.typeNimName = match.group(1) def to_string(self): - if self.nti: - nti = self.nti.value(gdb.newest_frame()) - enumStrings = [] - val = int(self.val) - i = 0 - while val > 0: - if (val & 1) == 1: - enumStrings.append(reprEnum(i, nti)) - val = val >> 1 - i += 1 - - return '{' + ', '.join(enumStrings) + '}' - else: - return str(int(self.val)) + # Remove the tySet from the type name + typ = gdb.lookup_type(self.val.type.name[6:]) + enumStrings = [] + val = int(self.val) + i = 0 + while val > 0: + if (val & 1) == 1: + enumStrings.append(reprEnum(i, typ)) + val = val >> 1 + i += 1 + + return '{' + ', '.join(enumStrings) + '}' ################################################################################ @@ -473,41 +469,81 @@ class NimHashSetPrinter: ################################################################################ -class NimSeqPrinter: - # the pointer is explicity part of the type. So it is part of - # ``pattern``. - pattern = re.compile(r'^tySequence_\w* \*$') +class NimSeq: + # Wrapper around sequences. + # This handles the differences between old and new runtime def __init__(self, val): self.val = val + # new runtime has sequences on stack, old has them on heap + self.new = val.type.code != gdb.TYPE_CODE_PTR + if self.new: + # Some seqs are just the content and to save repeating ourselves we do + # handle them here. Only thing that needs to check this is the len/data getters + self.isContent = val.type.name.endswith("Content") + + def __bool__(self): + if self.new: + return self.val is not None + else: + return bool(self.val) + + def __len__(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + else: + return int(self.val["len"]) + else: + return self.val["Sup"]["len"] + + @property + def data(self): + if self.new: + if self.isContent: + return self.val["data"] + elif self.val["p"]: + return self.val["p"]["data"] + else: + return self.val["data"] + + @property + def cap(self): + if not self: + return 0 + if self.new: + if self.isContent: + return int(self.val["cap"]) + elif self.val["p"]: + return int(self.val["p"]["cap"]) + else: + return 0 + return int(self.val['Sup']['reserved']) + +class NimSeqPrinter: + pattern = re.compile(r'^tySequence_\w*\s?\*?$') + + def __init__(self, val): + self.val = NimSeq(val) + def display_hint(self): return 'array' def to_string(self): - len = 0 - cap = 0 - if self.val: - len = int(self.val['Sup']['len']) - cap = int(self.val['Sup']['reserved']) - - return 'seq({0}, {1})'.format(len, cap) + return f'seq({len(self.val)}, {self.val.cap})' def children(self): if self.val: val = self.val - valType = val.type - length = int(val['Sup']['len']) + length = len(val) if length <= 0: return - dataType = valType['data'].type - data = val['data'] - - if self.val.type.name is None: - dataType = valType['data'].type.target().pointer() - data = val['data'].cast(dataType) + data = val.data inaccessible = False for i in range(length): @@ -563,7 +599,7 @@ class NimStringTablePrinter: def children(self): if self.val: - data = NimSeqPrinter(self.val['data'].dereference()) + data = NimSeqPrinter(self.val['data'].referenced_value()) for idxStr, entry in data.children(): if int(entry['Field0']) != 0: yield (idxStr + ".Field0", entry['Field0']) @@ -586,7 +622,7 @@ class NimTablePrinter: if self.val: counter = int(self.val['counter']) if self.val['data']: - capacity = int(self.val['data']['Sup']['len']) + capacity = NimSeq(self.val["data"]).cap return 'Table({0}, {1})'.format(counter, capacity) @@ -598,61 +634,18 @@ class NimTablePrinter: yield (idxStr + '.Field1', entry['Field1']) yield (idxStr + '.Field2', entry['Field2']) -################################################################ - -# this is untested, therefore disabled - -# class NimObjectPrinter: -# pattern = re.compile(r'^tyObject_.*$') - -# def __init__(self, val): -# self.val = val - -# def display_hint(self): -# return 'object' - -# def to_string(self): -# return str(self.val.type) - -# def children(self): -# if not self.val: -# yield "object", "<nil>" -# raise StopIteration - -# for (i, field) in enumerate(self.val.type.fields()): -# if field.type.code == gdb.TYPE_CODE_UNION: -# yield _union_field -# else: -# yield (field.name, self.val[field]) - -# def _union_field(self, i, field): -# rti = getNimRti(self.val.type.name) -# if rti is None: -# return (field.name, "UNION field can't be displayed without RTI") - -# node_sons = rti['node'].dereference()['sons'] -# prev_field = self.val.type.fields()[i - 1] - -# descriminant_node = None -# for i in range(int(node['len'])): -# son = node_sons[i].dereference() -# if son['name'].string("utf-8", "ignore") == str(prev_field.name): -# descriminant_node = son -# break -# if descriminant_node is None: -# raise ValueError("Can't find union descriminant field in object RTI") - -# if descriminant_node is None: raise ValueError("Can't find union field in object RTI") -# union_node = descriminant_node['sons'][int(self.val[prev_field])].dereference() -# union_val = self.val[field] +################################################################################ -# for f1 in union_val.type.fields(): -# for f2 in union_val[f1].type.fields(): -# if str(f2.name) == union_node['name'].string("utf-8", "ignore"): -# return (str(f2.name), union_val[f1][f2]) +class NimTuplePrinter: + pattern = re.compile(r"^tyTuple__([A-Za-z0-9]*)") -# raise ValueError("RTI is absent or incomplete, can't find union definition in RTI") + def __init__(self, val): + self.val = val + def to_string(self): + # We don't have field names so just print out the tuple as if it was anonymous + tupleValues = [str(self.val[field.name]) for field in self.val.type.fields()] + return f"({', '.join(tupleValues)})" ################################################################################ @@ -684,7 +677,7 @@ def makematcher(klass): return matcher def register_nim_pretty_printers_for_object(objfile): - nimMainSym = gdb.lookup_global_symbol("NimMain", gdb.SYMBOL_FUNCTIONS_DOMAIN) + nimMainSym = gdb.lookup_global_symbol("NimMain", gdb.SYMBOL_FUNCTION_DOMAIN) if nimMainSym and nimMainSym.symtab.objfile == objfile: print("set Nim pretty printers for ", objfile.filename) diff --git a/tools/debug/nimlldb.py b/tools/debug/nimlldb.py new file mode 100644 index 000000000..4bc4e771f --- /dev/null +++ b/tools/debug/nimlldb.py @@ -0,0 +1,1380 @@ +import lldb +from collections import OrderedDict +from typing import Union + + +def sbvaluegetitem(self: lldb.SBValue, name: Union[int, str]) -> lldb.SBValue: + if isinstance(name, str): + return self.GetChildMemberWithName(name) + else: + return self.GetChildAtIndex(name) + + +# Make this easier to work with +lldb.SBValue.__getitem__ = sbvaluegetitem + +NIM_IS_V2 = True + + +def get_nti(value: lldb.SBValue, nim_name=None): + name_split = value.type.name.split("_") + type_nim_name = nim_name or name_split[1] + id_string = name_split[-1].split(" ")[0] + + type_info_name = "NTI" + type_nim_name.lower() + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + type_info_name = "NTI" + "__" + id_string + "_" + nti = value.target.FindFirstGlobalVariable(type_info_name) + if not nti.IsValid(): + print(f"NimEnumPrinter: lookup global symbol: '{type_info_name}' failed for {value.type.name}.\n") + return type_nim_name, nti + + +def enum_to_string(value: lldb.SBValue, int_val=None, nim_name=None): + tname = nim_name or value.type.name.split("_")[1] + + enum_val = value.signed + if int_val is not None: + enum_val = int_val + + default_val = f"{tname}.{str(enum_val)}" + + fn_syms = value.target.FindFunctions("reprEnum") + if not fn_syms.GetSize() > 0: + return default_val + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + if arg_types.GetSize() < 2: + return default_val + + arg1_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + arg2_type: lldb.SBType = arg_types.GetTypeAtIndex(1) + + ty_info_name, nti = get_nti(value, nim_name=tname) + + if not nti.IsValid(): + return default_val + + call = f"{fn.name}(({arg1_type.name}){enum_val}, ({arg2_type.name})" + str(nti.GetLoadAddress()) + ");" + + res = executeCommand(call) + + if res.error.fail: + return default_val + + return f"{tname}.{res.summary[1:-1]}" + + +def to_string(value: lldb.SBValue): + # For getting NimStringDesc * value + value = value.GetNonSyntheticValue() + + # Check if data pointer is Null + if value.type.is_pointer and value.unsigned == 0: + return None + + size = int(value["Sup"]["len"].unsigned) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + data = value["data"] + + # Check if first element is NULL + base_data_type = value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(value.target.FindFirstType("char").GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def to_stringV2(value: lldb.SBValue): + # For getting NimStringV2 value + value = value.GetNonSyntheticValue() + + data = value["p"]["data"] + + # Check if data pointer is Null + if value["p"].unsigned == 0: + return None + + size = int(value["len"].signed) + + if size == 0: + return "" + + if size > 2**14: + return "... (too long) ..." + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return None + + cast = data.Cast(base_data_type.GetArrayType(size)) + return bytearray(cast.data.uint8s).decode("utf-8") + + +def NimString(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + if NIM_IS_V2: + res = to_stringV2(value) + else: + res = to_string(value) + + if res is not None: + return f'"{res}"' + else: + return "nil" + + +def rope_helper(value: lldb.SBValue) -> str: + value = value.GetNonSyntheticValue() + if value.type.is_pointer and value.unsigned == 0: + return "" + + if value["length"].unsigned == 0: + return "" + + if NIM_IS_V2: + str_val = to_stringV2(value["data"]) + else: + str_val = to_string(value["data"]) + + if str_val is None: + str_val = "" + + return rope_helper(value["left"]) + str_val + rope_helper(value["right"]) + + +def Rope(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + rope_str = rope_helper(value) + + if len(rope_str) == 0: + rope_str = "nil" + else: + rope_str = f'"{rope_str}"' + + return f"Rope({rope_str})" + + +def NCSTRING(value: lldb.SBValue, internal_dict=None): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + ty = value.Dereference().type + val = value.target.CreateValueFromAddress( + value.name or "temp", lldb.SBAddress(value.unsigned, value.target), ty + ).AddressOf() + return val.summary + + +def ObjectV2(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + orig_value = value.GetNonSyntheticValue() + if orig_value.type.is_pointer and orig_value.unsigned == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + while orig_value.type.is_pointer: + orig_value = orig_value.Dereference() + + if "_" in orig_value.type.name: + obj_name = orig_value.type.name.split("_")[1].replace("colonObjectType", "") + else: + obj_name = orig_value.type.name + + num_children = value.num_children + fields = [] + + for i in range(num_children): + fields.append(f"{value[i].name}: {value[i].summary}") + + res = f"{obj_name}(" + ", ".join(fields) + ")" + return res + + +def Number(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + if value.type.is_pointer and value.signed == 0: + return "nil" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.signed) + + +def Float(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def UnsignedNumber(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.unsigned) + + +def Bool(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str(value.value) + + +def CharArray(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return str([f"'{char}'" for char in value.uint8s]) + + +def Array(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + value = value.GetNonSyntheticValue() + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + value = value.GetNonSyntheticValue() + return "[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +def Tuple(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + while value.type.is_pointer: + value = value.Dereference() + + num_children = value.num_children + + fields = [] + + for i in range(num_children): + key = value[i].name + val = value[i].summary + if key.startswith("Field"): + fields.append(f"{val}") + else: + fields.append(f"{key}: {val}") + + return "(" + ", ".join(fields) + f")" + + +def is_local(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if line.file == decl.file and decl.line != 0: + return True + + return False + + +def is_in_scope(value: lldb.SBValue) -> bool: + line: lldb.SBLineEntry = value.frame.GetLineEntry() + decl: lldb.SBDeclaration = value.GetDeclaration() + + if is_local(value) and decl.line < line.line: + return True + + return False + + +def Enum(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_value_summary(value) + if custom_summary is not None: + return custom_summary + + return enum_to_string(value) + + +def EnumSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.summary) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Set(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + vals = [] + max_vals = 7 + for child in value.children: + vals.append(child.value) + if len(vals) > max_vals: + vals.append("...") + break + + return "{" + ", ".join(vals) + "}" + + +def Table(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + return "Table({" + ", ".join(fields) + "})" + + +def HashSet(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if custom_summary is not None: + return custom_summary + + fields = [] + + for i in range(value.num_children): + fields.append(f"{value[i].summary}") + + return "HashSet({" + ", ".join(fields) + "})" + + +def StringTable(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + fields = [] + + for i in range(value.num_children - 1): + key = value[i].name + val = value[i].summary + fields.append(f"{key}: {val}") + + mode = value[value.num_children - 1].summary + + return "StringTable({" + ", ".join(fields) + f"}}, mode={mode})" + + +def Sequence(value: lldb.SBValue, internal_dict): + if is_local(value): + if not is_in_scope(value): + return "undefined" + + custom_summary = get_custom_summary(value) + if not custom_summary is None: + return custom_summary + + return "@[" + ", ".join([value[i].summary for i in range(value.num_children)]) + "]" + + +class StringChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + if not NIM_IS_V2: + self.data_type = self.value.target.FindFirstType("char") + + self.first_element: lldb.SBValue + self.update() + self.count = 0 + + def num_children(self): + return self.count + + def get_child_index(self, name): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data_size + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> int: + if NIM_IS_V2: + if self.value["p"].unsigned == 0: + return 0 + + size = int(self.value["len"].signed) + + if size == 0: + return 0 + + data = self.value["p"]["data"] + + # Check if first element is NULL + base_data_type = data.type.GetArrayElementType().GetTypedefedType() + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + else: + if self.value.type.is_pointer and self.value.unsigned == 0: + return 0 + + size = int(self.value["Sup"]["len"].unsigned) + + if size == 0: + return 0 + + data = self.value["data"] + + # Check if first element is NULL + base_data_type = self.value.target.FindFirstType("char") + cast = data.Cast(base_data_type) + + if cast.unsigned == 0: + return 0 + + return size + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + + data = self.get_data() + size = self.get_len() + + self.count = size + self.first_element = data + + if NIM_IS_V2: + self.data_type = data.type.GetArrayElementType().GetTypedefedType() + + self.data_size = self.data_type.GetByteSize() + + def has_children(self): + return bool(self.num_children()) + + +class ArrayChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.update() + + def num_children(self): + return self.has_children() and self.value.num_children + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.value[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def update(self): + if not self.has_children(): + return + + self.first_element = self.value[0] + self.data_type = self.value.type.GetArrayElementType() + + def has_children(self): + if is_local(self.value): + if not is_in_scope(self.value): + return False + return bool(self.value.num_children) + + +class SeqChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.count = 0 + self.update() + + def num_children(self): + return self.count + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + offset = index * self.data[index].GetByteSize() + return self.first_element.CreateChildAtOffset("[" + str(index) + "]", offset, self.data_type) + + def get_data(self) -> lldb.SBValue: + return self.value["p"]["data"] if NIM_IS_V2 else self.value["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["len"] if NIM_IS_V2 else self.value["Sup"]["len"] + + def update(self): + self.count = 0 + + if is_local(self.value): + if not is_in_scope(self.value): + return + + self.count = self.get_len().unsigned + + if not self.has_children(): + return + + data = self.get_data() + self.data_type = data.type.GetArrayElementType() + + self.data = data.Cast(self.data_type.GetArrayType(self.num_children())) + self.first_element = self.data + + def has_children(self): + return bool(self.num_children()) + + +class ObjectChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.data_type: lldb.SBType + self.first_element: lldb.SBValue + self.data: lldb.SBValue + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.children) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def populate_children(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + stack = [self.value.GetNonSyntheticValue()] + + index = 0 + + while stack: + cur_val = stack.pop() + if cur_val.type.is_pointer and cur_val.unsigned == 0: + continue + + while cur_val.type.is_pointer: + cur_val = cur_val.Dereference() + + # Add super objects if they exist + if cur_val.num_children > 0 and cur_val[0].name == "Sup" and cur_val[0].type.name.startswith("tyObject"): + stack.append(cur_val[0]) + + for child in cur_val.children: + child = child.GetNonSyntheticValue() + if child.name == "Sup": + continue + self.children[child.name] = index + self.child_list.append(child) + index += 1 + + def update(self): + self.populate_children() + + def has_children(self): + return bool(self.num_children()) + + +class HashSetChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + child = key.CreateValueFromAddress(f"[{str(index)}]", key.GetLoadAddress(), key.GetType()) + index += 1 + + self.child_list.append(child) + + def has_children(self): + return bool(self.num_children()) + + +class SetCharChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType("char") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + cur_pos = 0 + for child in self.value.children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = self.value.synthetic_child_from_data(f"[{len(self.child_list)}]", data, self.ty) + self.child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + def has_children(self): + return bool(self.num_children()) + + +def create_set_children(value: lldb.SBValue, child_type: lldb.SBType, starting_pos: int) -> list[lldb.SBValue]: + child_list: list[lldb.SBValue] = [] + cur_pos = starting_pos + + if value.num_children > 0: + children = value.children + else: + children = [value] + + for child in children: + child_val = child.signed + if child_val != 0: + temp = child_val + num_bits = 8 + while temp != 0: + is_set = temp & 1 + if is_set == 1: + data = lldb.SBData.CreateDataFromInt(cur_pos) + child = value.synthetic_child_from_data(f"[{len(child_list)}]", data, child_type) + child_list.append(child) + temp = temp >> 1 + cur_pos += 1 + num_bits -= 1 + cur_pos += num_bits + else: + cur_pos += 8 + + return child_list + + +class SetIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NI64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + bits = self.value.GetByteSize() * 8 + starting_pos = -(bits // 2) + self.child_list = create_set_children(self.value, self.ty, starting_pos) + + def has_children(self): + return bool(self.num_children()) + + +class SetUIntChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(f"NU64") + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class SetEnumChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.ty = self.value.target.FindFirstType(self.value.type.name.replace("tySet_", "")) + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return int(name.lstrip("[").rstrip("]")) + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + if is_local(self.value): + if not is_in_scope(self.value): + return + self.child_list = create_set_children(self.value, self.ty, starting_pos=0) + + def has_children(self): + return bool(self.num_children()) + + +class TableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.child_list = [] + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[0].unsigned) + if field0 == 0: + continue + key = el[1] + val = el[2] + key_summary = key.summary + child = self.value.CreateValueFromAddress(key_summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key_summary] = index + index += 1 + + def has_children(self): + return bool(self.num_children()) + + +class StringTableChildrenProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value = value + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def get_data(self) -> lldb.SBValue: + return self.value["data"]["p"]["data"] if NIM_IS_V2 else self.value["data"]["data"] + + def get_len(self) -> lldb.SBValue: + return self.value["data"]["len"] if NIM_IS_V2 else self.value["data"]["Sup"]["len"] + + def update(self): + self.children.clear() + self.child_list = [] + + if is_local(self.value): + if not is_in_scope(self.value): + return + + tuple_len = int(self.get_len().unsigned) + tuple = self.get_data() + + base_data_type = tuple.type.GetArrayElementType() + + cast = tuple.Cast(base_data_type.GetArrayType(tuple_len)) + + index = 0 + for i in range(tuple_len): + el = cast[i] + field0 = int(el[2].unsigned) + if field0 == 0: + continue + key = el[0] + val = el[1] + child = val.CreateValueFromAddress(key.summary, val.GetLoadAddress(), val.GetType()) + self.child_list.append(child) + self.children[key.summary] = index + index += 1 + + self.child_list.append(self.value["mode"]) + self.children["mode"] = index + + def has_children(self): + return bool(self.num_children()) + + +class LLDBDynamicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + value = value.GetNonSyntheticValue() + self.value: lldb.SBValue = value[0] + self.children: OrderedDict[str, int] = OrderedDict() + self.child_list: list[lldb.SBValue] = [] + + while self.value.type.is_pointer: + self.value = self.value.Dereference() + + self.update() + + def num_children(self): + return len(self.child_list) + + def get_child_index(self, name: str): + return self.children[name] + + def get_child_at_index(self, index): + return self.child_list[index] + + def update(self): + self.children.clear() + self.child_list = [] + + for i, child in enumerate(self.value.children): + name = child.name.strip('"') + new_child = child.CreateValueFromAddress(name, child.GetLoadAddress(), child.GetType()) + + self.children[name] = i + self.child_list.append(new_child) + + def has_children(self): + return bool(self.num_children()) + + +class LLDBBasicObjectProvider: + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = value + + def num_children(self): + if self.value is not None: + return self.value.num_children + return 0 + + def get_child_index(self, name: str): + return self.value.GetIndexOfChildWithName(name) + + def get_child_at_index(self, index): + return self.value.GetChildAtIndex(index) + + def update(self): + pass + + def has_children(self): + return self.num_children() > 0 + + +class CustomObjectChildrenProvider: + """ + This children provider handles values returned from lldbDebugSynthetic* + Nim procedures + """ + + def __init__(self, value: lldb.SBValue, internalDict): + self.value: lldb.SBValue = get_custom_synthetic(value) or value + if "lldbdynamicobject" in self.value.type.name.lower(): + self.provider = LLDBDynamicObjectProvider(self.value, internalDict) + else: + self.provider = LLDBBasicObjectProvider(self.value, internalDict) + + def num_children(self): + return self.provider.num_children() + + def get_child_index(self, name: str): + return self.provider.get_child_index(name) + + def get_child_at_index(self, index): + return self.provider.get_child_at_index(index) + + def update(self): + self.provider.update() + + def has_children(self): + return self.provider.has_children() + + +def echo(debugger: lldb.SBDebugger, command: str, result, internal_dict): + debugger.HandleCommand("po " + command) + + +SUMMARY_FUNCTIONS: dict[str, lldb.SBFunction] = {} +SYNTHETIC_FUNCTIONS: dict[str, lldb.SBFunction] = {} + + +def get_custom_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SUMMARY_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + command = f"{fn.name}(({first_type.name})" + str(value.GetLoadAddress()) + ");" + else: + command = f"{fn.name}(*({first_type.GetPointerType().name})" + str(value.GetLoadAddress()) + ");" + + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_value_summary(value: lldb.SBValue) -> Union[str, None]: + """Get a custom summary if a function exists for it""" + + fn: lldb.SBFunction = SUMMARY_FUNCTIONS.get(value.type.name) + if fn is None: + return None + + command = f"{fn.name}(({value.type.name})" + str(value.signed) + ");" + res = executeCommand(command) + + if res.error.fail: + return None + + return res.summary.strip('"') + + +def get_custom_synthetic(value: lldb.SBValue) -> Union[lldb.SBValue, None]: + """Get a custom synthetic object if a function exists for it""" + value = value.GetNonSyntheticValue() + if value.GetAddress().GetOffset() == 0: + return None + + base_type = get_base_type(value.type) + + fn = SYNTHETIC_FUNCTIONS.get(base_type.name) + if fn is None: + return None + + fn_type: lldb.SBType = fn.type + + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + first_type = arg_types.GetTypeAtIndex(0) + + while value.type.is_pointer: + value = value.Dereference() + + if first_type.is_pointer: + first_arg = f"({first_type.name}){value.GetLoadAddress()}" + else: + first_arg = f"*({first_type.GetPointerType().name}){value.GetLoadAddress()}" + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + ret_type = arg_types.GetTypeAtIndex(1) + ret_type = get_base_type(ret_type) + + command = f""" + {ret_type.name} lldbT; + nimZeroMem((void*)(&lldbT), sizeof({ret_type.name})); + {fn.name}(({first_arg}), (&lldbT)); + lldbT; + """ + else: + command = f"{fn.name}({first_arg});" + + res = executeCommand(command) + + if res.error.fail: + print(res.error) + return None + + return res + + +def get_base_type(ty: lldb.SBType) -> lldb.SBType: + """Get the base type of the type""" + temp = ty + while temp.IsPointerType(): + temp = temp.GetPointeeType() + return temp + + +def use_base_type(ty: lldb.SBType) -> bool: + types_to_check = [ + "NF", + "NF32", + "NF64", + "NI", + "NI8", + "NI16", + "NI32", + "NI64", + "bool", + "NIM_BOOL", + "NU", + "NU8", + "NU16", + "NU32", + "NU64", + ] + + for type_to_check in types_to_check: + if ty.name.startswith(type_to_check): + return False + + return True + + +def breakpoint_function_wrapper(frame: lldb.SBFrame, bp_loc, internal_dict): + """This allows function calls to Nim for custom object summaries and synthetic children""" + debugger = lldb.debugger + + global SUMMARY_FUNCTIONS + global SYNTHETIC_FUNCTIONS + + global NIM_IS_V2 + + for tname, fn in SYNTHETIC_FUNCTIONS.items(): + debugger.HandleCommand(f"type synthetic delete -w nim {tname}") + + SUMMARY_FUNCTIONS = {} + SYNTHETIC_FUNCTIONS = {} + + target: lldb.SBTarget = debugger.GetSelectedTarget() + + NIM_IS_V2 = target.FindFirstType("TNimTypeV2").IsValid() + + module = frame.GetSymbolContext(lldb.eSymbolContextModule).module + + for sym in module: + if ( + not sym.name.startswith("lldbDebugSummary") + and not sym.name.startswith("lldbDebugSynthetic") + and not sym.name.startswith("dollar___") + ): + continue + + fn_syms: lldb.SBSymbolContextList = target.FindFunctions(sym.name) + if not fn_syms.GetSize() > 0: + continue + + fn_sym: lldb.SBSymbolContext = fn_syms.GetContextAtIndex(0) + + fn: lldb.SBFunction = fn_sym.function + fn_type: lldb.SBType = fn.type + arg_types: lldb.SBTypeList = fn_type.GetFunctionArgumentTypes() + + if arg_types.GetSize() > 1 and fn.GetArgumentName(1) == "Result": + pass # don't continue + elif arg_types.GetSize() != 1: + continue + + arg_type: lldb.SBType = arg_types.GetTypeAtIndex(0) + if use_base_type(arg_type): + arg_type = get_base_type(arg_type) + + if sym.name.startswith("lldbDebugSummary") or sym.name.startswith("dollar___"): + SUMMARY_FUNCTIONS[arg_type.name] = fn + elif sym.name.startswith("lldbDebugSynthetic"): + SYNTHETIC_FUNCTIONS[arg_type.name] = fn + debugger.HandleCommand( + f"type synthetic add -w nim -l {__name__}.CustomObjectChildrenProvider {arg_type.name}" + ) + + +def executeCommand(command, *args): + debugger = lldb.debugger + process = debugger.GetSelectedTarget().GetProcess() + frame: lldb.SBFrame = process.GetSelectedThread().GetSelectedFrame() + + expr_options = lldb.SBExpressionOptions() + expr_options.SetIgnoreBreakpoints(False) + expr_options.SetFetchDynamicValue(lldb.eDynamicCanRunTarget) + expr_options.SetTimeoutInMicroSeconds(30 * 1000 * 1000) # 30 second timeout + expr_options.SetTryAllThreads(True) + expr_options.SetUnwindOnError(False) + expr_options.SetGenerateDebugInfo(True) + expr_options.SetLanguage(lldb.eLanguageTypeC) + expr_options.SetCoerceResultToId(True) + res = frame.EvaluateExpression(command, expr_options) + + return res + + +def __lldb_init_module(debugger, internal_dict): + # fmt: off + debugger.HandleCommand(f"breakpoint command add -F {__name__}.breakpoint_function_wrapper --script-type python 1") + debugger.HandleCommand(f"type summary add -w nim -n sequence -F {__name__}.Sequence -x tySequence_+[[:alnum:]]+$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SeqChildrenProvider -x tySequence_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n chararray -F {__name__}.CharArray -x char\s+[\d+]") + debugger.HandleCommand(f"type summary add -w nim -n array -F {__name__}.Array -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ArrayChildrenProvider -x tyArray_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n string -F {__name__}.NimString NimStringDesc") + + debugger.HandleCommand(f"type summary add -w nim -n stringv2 -F {__name__}.NimString -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringV2$") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringChildrenProvider -x NimStringDesc$") + + debugger.HandleCommand(f"type summary add -w nim -n cstring -F {__name__}.NCSTRING NCSTRING") + + debugger.HandleCommand(f"type summary add -w nim -n object -F {__name__}.ObjectV2 -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.ObjectChildrenProvider -x tyObject_+[[:alnum:]]+_+[[:alnum:]]+$") + + debugger.HandleCommand(f"type summary add -w nim -n tframe -F {__name__}.ObjectV2 -x TFrame$") + + debugger.HandleCommand(f"type summary add -w nim -n rootobj -F {__name__}.ObjectV2 -x RootObj$") + + debugger.HandleCommand(f"type summary add -w nim -n enum -F {__name__}.Enum -x tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n hashset -F {__name__}.HashSet -x tyObject_+HashSet_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.HashSetChildrenProvider -x tyObject_+HashSet_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n rope -F {__name__}.Rope -x tyObject_+Rope[[:alnum:]]+_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n setuint -F {__name__}.Set -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetIntChildrenProvider -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setint -F {__name__}.Set -x tySet_+tyInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setuint2 -F {__name__}.Set -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyUInt[0-9]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetUIntChildrenProvider -x tySet_+tyInt_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setenum -F {__name__}.EnumSet -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetEnumChildrenProvider -x tySet_+tyEnum_+[[:alnum:]]+_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n setchar -F {__name__}.Set -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.SetCharChildrenProvider -x tySet_+tyChar_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n table -F {__name__}.Table -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.TableChildrenProvider -x tyObject_+Table_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n stringtable -F {__name__}.StringTable -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type synthetic add -w nim -l {__name__}.StringTableChildrenProvider -x tyObject_+StringTableObj_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple2 -F {__name__}.Tuple -x tyObject_+Tuple_+[[:alnum:]]+") + debugger.HandleCommand(f"type summary add -w nim -n tuple -F {__name__}.Tuple -x tyTuple_+[[:alnum:]]+") + + debugger.HandleCommand(f"type summary add -w nim -n float -F {__name__}.Float NF") + debugger.HandleCommand(f"type summary add -w nim -n float32 -F {__name__}.Float NF32") + debugger.HandleCommand(f"type summary add -w nim -n float64 -F {__name__}.Float NF64") + debugger.HandleCommand(f"type summary add -w nim -n integer -F {__name__}.Number -x NI") + debugger.HandleCommand(f"type summary add -w nim -n integer8 -F {__name__}.Number -x NI8") + debugger.HandleCommand(f"type summary add -w nim -n integer16 -F {__name__}.Number -x NI16") + debugger.HandleCommand(f"type summary add -w nim -n integer32 -F {__name__}.Number -x NI32") + debugger.HandleCommand(f"type summary add -w nim -n integer64 -F {__name__}.Number -x NI64") + debugger.HandleCommand(f"type summary add -w nim -n bool -F {__name__}.Bool -x bool") + debugger.HandleCommand(f"type summary add -w nim -n bool2 -F {__name__}.Bool -x NIM_BOOL") + debugger.HandleCommand(f"type summary add -w nim -n uinteger -F {__name__}.UnsignedNumber -x NU") + debugger.HandleCommand(f"type summary add -w nim -n uinteger8 -F {__name__}.UnsignedNumber -x NU8") + debugger.HandleCommand(f"type summary add -w nim -n uinteger16 -F {__name__}.UnsignedNumber -x NU16") + debugger.HandleCommand(f"type summary add -w nim -n uinteger32 -F {__name__}.UnsignedNumber -x NU32") + debugger.HandleCommand(f"type summary add -w nim -n uinteger64 -F {__name__}.UnsignedNumber -x NU64") + debugger.HandleCommand("type category enable nim") + debugger.HandleCommand(f"command script add -f {__name__}.echo echo") + # fmt: on diff --git a/tools/deps.nim b/tools/deps.nim index 8ed95b59a..e43f7a2b4 100644 --- a/tools/deps.nim +++ b/tools/deps.nim @@ -1,6 +1,9 @@ -import os, uri, strformat, strutils +import std/[os, uri, strformat, strutils] import std/private/gitutils +when defined(nimPreviewSlimSystem): + import std/assertions + proc exec(cmd: string) = echo "deps.cmd: " & cmd let status = execShellCmd(cmd) @@ -22,14 +25,19 @@ proc cloneDependency*(destDirBase: string, url: string, commit = commitHead, let name = p.splitFile.name var destDir = destDirBase if appendRepoName: destDir = destDir / name - let destDir2 = destDir.quoteShell + let quotedDestDir = destDir.quoteShell if not dirExists(destDir): # note: old code used `destDir / .git` but that wouldn't prevent git clone # from failing - execRetry fmt"git clone -q {url} {destDir2}" + execRetry fmt"git clone -q {url} {quotedDestDir}" if isGitRepo(destDir): - execRetry fmt"git -C {destDir2} fetch -q" - exec fmt"git -C {destDir2} checkout -q {commit}" + let oldDir = getCurrentDir() + setCurrentDir(destDir) + try: + execRetry "git fetch -q" + exec fmt"git checkout -q {commit}" + finally: + setCurrentDir(oldDir) elif allowBundled: discard "this dependency was bundled with Nim, don't do anything" else: diff --git a/tools/detect/detect.nim b/tools/detect/detect.nim index 841b3a675..ed9438494 100644 --- a/tools/detect/detect.nim +++ b/tools/detect/detect.nim @@ -12,7 +12,10 @@ # The second one is more portable, and less semantically correct. It only works # when there's a backing C compiler available as well, preventing standalone # compilation. -import os, strutils +import std/[os, strutils] + +when defined(nimPreviewSlimSystem): + import std/syncio when defined(openbsd) or defined(freebsd) or defined(netbsd): const @@ -615,6 +618,7 @@ v("MAP_POPULATE", no_other = true) header("<sys/resource.h>") v("RLIMIT_NOFILE") +v("RLIMIT_STACK") header("<sys/select.h>") v("FD_SETSIZE") @@ -626,6 +630,7 @@ v("MSG_EOR") v("MSG_OOB") v("SCM_RIGHTS") v("SO_ACCEPTCONN") +v("SO_BINDTODEVICE") v("SO_BROADCAST") v("SO_DEBUG") v("SO_DONTROUTE") diff --git a/tools/dochack/dochack.nim b/tools/dochack/dochack.nim index 4a491cf88..946945346 100644 --- a/tools/dochack/dochack.nim +++ b/tools/dochack/dochack.nim @@ -1,49 +1,52 @@ import dom import fuzzysearch +import std/[jsfetch, asyncjs] -proc textContent(e: Element): cstring {. - importcpp: "#.textContent", nodecl.} -proc textContent(e: Node): cstring {. - importcpp: "#.textContent", nodecl.} +proc setTheme(theme: cstring) {.exportc.} = + document.documentElement.setAttribute("data-theme", theme) + window.localStorage.setItem("theme", theme) -proc tree(tag: string; kids: varargs[Element]): Element = +# set `data-theme` attribute early to prevent white flash +setTheme: + let t = window.localStorage.getItem("theme") + if t.isNil: cstring"auto" else: t + +proc onDOMLoaded(e: Event) {.exportc.} = + # set theme select value + document.getElementById("theme-select").value = window.localStorage.getItem("theme") + + for pragmaDots in document.getElementsByClassName("pragmadots"): + pragmaDots.onclick = proc (event: Event) = + # Hide tease + event.target.parentNode.style.display = "none" + # Show actual + event.target.parentNode.nextSibling.style.display = "inline" + + +proc tree(tag: cstring; kids: varargs[Element]): Element = result = document.createElement tag for k in kids: result.appendChild k proc add(parent, kid: Element) = - if parent.nodeName == cstring"TR" and ( - kid.nodeName == cstring"TD" or kid.nodeName == cstring"TH"): + if parent.nodeName == "TR" and (kid.nodeName == "TD" or kid.nodeName == "TH"): let k = document.createElement("TD") appendChild(k, kid) appendChild(parent, k) else: appendChild(parent, kid) -proc setClass(e: Element; value: string) = +proc setClass(e: Element; value: cstring) = e.setAttribute("class", value) -proc text(s: string): Element = cast[Element](document.createTextNode(s)) proc text(s: cstring): Element = cast[Element](document.createTextNode(s)) -proc getElementById(id: cstring): Element {.importc: "document.getElementById", nodecl.} - proc replaceById(id: cstring; newTree: Node) = - let x = getElementById(id) + let x = document.getElementById(id) x.parentNode.replaceChild(newTree, x) newTree.id = id -proc findNodeWith(x: Element; tag, content: cstring): Element = - if x.nodeName == tag and x.textContent == content: - return x - for i in 0..<x.len: - let it = x[i] - let y = findNodeWith(it, tag, content) - if y != nil: return y - return nil - proc clone(e: Element): Element {.importcpp: "#.cloneNode(true)", nodecl.} -proc parent(e: Element): Element {.importcpp: "#.parentNode", nodecl.} proc markElement(x: Element) {.importcpp: "#.__karaxMarker__ = true", nodecl.} proc isMarked(x: Element): bool {. importcpp: "#.hasOwnProperty('__karaxMarker__')", nodecl.} @@ -52,20 +55,13 @@ proc title(x: Element): cstring {.importcpp: "#.title", nodecl.} proc sort[T](x: var openArray[T]; cmp: proc(a, b: T): int) {.importcpp: "#.sort(#)", nodecl.} -proc parentWith(x: Element; tag: cstring): Element = - result = x.parent - while result.nodeName != tag: - result = result.parent - if result == nil: return nil - proc extractItems(x: Element; items: var seq[Element]) = if x == nil: return - if x.nodeName == cstring"A": + if x.nodeName == "A": items.add x else: for i in 0..<x.len: - let it = x[i] - extractItems(it, items) + extractItems(x[i], items) # HTML trees are so shitty we transform the TOC into a decent # data-structure instead and work on that. @@ -76,16 +72,14 @@ type sortId: int doSort: bool -proc extractItems(x: TocEntry; heading: cstring; - items: var seq[Element]) = +proc extractItems(x: TocEntry; heading: cstring; items: var seq[Element]) = if x == nil: return if x.heading != nil and x.heading.textContent == heading: for i in 0..<x.kids.len: items.add x.kids[i].heading else: - for i in 0..<x.kids.len: - let it = x.kids[i] - extractItems(it, heading, items) + for k in x.kids: + extractItems(k, heading, items) proc toHtml(x: TocEntry; isRoot=false): Element = if x == nil: return nil @@ -119,31 +113,21 @@ proc toHtml(x: TocEntry; isRoot=false): Element = if ul.len != 0: result.add ul if result.len == 0: result = nil -#proc containsWord(a, b: cstring): bool {.asmNoStackFrame.} = - #{.emit: """ - #var escaped = `b`.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&"); - #return new RegExp("\\b" + escaped + "\\b").test(`a`); - #""".} - -proc isWhitespace(text: cstring): bool {.asmNoStackFrame.} = - {.emit: """ - return !/[^\s]/.test(`text`); - """.} +proc isWhitespace(text: cstring): bool {.importcpp: r"!/\S/.test(#)".} proc isWhitespace(x: Element): bool = - x.nodeName == cstring"#text" and x.textContent.isWhitespace or - x.nodeName == cstring"#comment" + x.nodeName == "#text" and x.textContent.isWhitespace or x.nodeName == "#comment" proc toToc(x: Element; father: TocEntry) = - if x.nodeName == cstring"UL": + if x.nodeName == "UL": let f = TocEntry(heading: nil, kids: @[], sortId: father.kids.len) var i = 0 while i < x.len: var nxt = i+1 while nxt < x.len and x[nxt].isWhitespace: inc nxt - if nxt < x.len and x[i].nodeName == cstring"LI" and x[i].len == 1 and - x[nxt].nodeName == cstring"UL": + if nxt < x.len and x[i].nodeName == "LI" and x[i].len == 1 and + x[nxt].nodeName == "UL": let e = TocEntry(heading: x[i][0], kids: @[], sortId: f.kids.len) let it = x[nxt] for j in 0..<it.len: @@ -156,13 +140,12 @@ proc toToc(x: Element; father: TocEntry) = father.kids.add f elif isWhitespace(x): discard - elif x.nodeName == cstring"LI": + elif x.nodeName == "LI": var idx: seq[int] = @[] for i in 0 ..< x.len: if not x[i].isWhitespace: idx.add i - if idx.len == 2 and x[idx[1]].nodeName == cstring"UL": - let e = TocEntry(heading: x[idx[0]], kids: @[], - sortId: father.kids.len) + if idx.len == 2 and x[idx[1]].nodeName == "UL": + let e = TocEntry(heading: x[idx[0]], kids: @[], sortId: father.kids.len) let it = x[idx[1]] for j in 0..<it.len: toToc(it[j], e) @@ -171,31 +154,25 @@ proc toToc(x: Element; father: TocEntry) = for i in 0..<x.len: toToc(x[i], father) else: - father.kids.add TocEntry(heading: x, kids: @[], - sortId: father.kids.len) + father.kids.add TocEntry(heading: x, kids: @[], sortId: father.kids.len) proc tocul(x: Element): Element = # x is a 'ul' element result = tree("UL") for i in 0..<x.len: let it = x[i] - if it.nodeName == cstring"LI": + if it.nodeName == "LI": result.add it.clone - elif it.nodeName == cstring"UL": + elif it.nodeName == "UL": result.add tocul(it) -proc getSection(toc: Element; name: cstring): Element = - let sec = findNodeWith(toc, "A", name) - if sec != nil: - result = sec.parentWith("LI") - proc uncovered(x: TocEntry): TocEntry = if x.kids.len == 0 and x.heading != nil: return if not isMarked(x.heading): x else: nil result = TocEntry(heading: x.heading, kids: @[], sortId: x.sortId, doSort: x.doSort) - for i in 0..<x.kids.len: - let y = uncovered(x.kids[i]) + for k in x.kids: + let y = uncovered(k) if y != nil: result.kids.add y if result.kids.len == 0: result = nil @@ -214,9 +191,8 @@ proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = t.markElement() for p in procs: if not isMarked(p): - let xx = getElementsByClass(p.parent, cstring"attachedType") + let xx = getElementsByClass(p.parentNode, "attachedType") if xx.len == 1 and xx[0].textContent == t.textContent: - #kout(cstring"found ", p.nodeName) let q = tree("A", text(p.title)) q.setAttr("href", p.getAttribute("href")) c.kids.add TocEntry(heading: q, kids: @[]) @@ -227,15 +203,13 @@ proc buildToc(orig: TocEntry; types, procs: seq[Element]): TocEntry = var alternative: Element proc togglevis(d: Element) = - asm """ - if (`d`.style.display == 'none') - `d`.style.display = 'inline'; - else - `d`.style.display = 'none'; - """ + if d.style.display == "none": + d.style.display = "inline" + else: + d.style.display = "none" proc groupBy*(value: cstring) {.exportc.} = - let toc = getElementById("toc-list") + let toc = document.getElementById("toc-list") if alternative.isNil: var tt = TocEntry(heading: nil, kids: @[]) toToc(toc, tt) @@ -255,17 +229,16 @@ proc groupBy*(value: cstring) {.exportc.} = let ntoc = buildToc(tt, types, procs) let x = toHtml(ntoc, isRoot=true) alternative = tree("DIV", x) - if value == cstring"type": + if value == "type": replaceById("tocRoot", alternative) else: replaceById("tocRoot", tree("DIV")) - togglevis(getElementById"toc-list") + togglevis(document.getElementById"toc-list") var db: seq[Node] contents: seq[cstring] -template normalize(x: cstring): cstring = x.toLower.replace("_", "") proc escapeCString(x: var cstring) = # Original strings are already escaped except HTML tags, so @@ -280,31 +253,14 @@ proc escapeCString(x: var cstring) = proc dosearch(value: cstring): Element = if db.len == 0: - var stuff: Element - {.emit: """ - var request = new XMLHttpRequest(); - request.open("GET", "theindex.html", false); - request.send(null); - - var doc = document.implementation.createHTMLDocument("theindex"); - doc.documentElement.innerHTML = request.responseText; - - //parser=new DOMParser(); - //doc=parser.parseFromString("<html></html>", "text/html"); - - `stuff` = doc.documentElement; - """.} - db = stuff.getElementsByClass"reference" - contents = @[] - for ahref in db: - contents.add ahref.getAttribute("data-doc-search-tag") + return let ul = tree("UL") result = tree("DIV") result.setClass"search_results" var matches: seq[(Node, int)] = @[] for i in 0..<db.len: let c = contents[i] - if c == cstring"Examples" or c == cstring"PEG construction": + if c == "Examples" or c == "PEG construction": # Some manual exclusions. # Ideally these should be fixed in the index to be more # descriptive of what they are. @@ -324,20 +280,103 @@ proc dosearch(value: cstring): Element = result.add tree("B", text"search results") result.add ul -var oldtoc: Element -var timer: Timeout +proc loadIndex() {.async.} = + ## Loads theindex.html to enable searching + let + indexURL = document.getElementById("indexLink").getAttribute("href") + # Get root of project documentation by cutting off theindex.html from index href + rootURL = ($indexURL)[0 ..< ^"theindex.html".len] + var resp = fetch(indexURL).await().text().await() + # Convert into element so we can use DOM functions to parse the html + var indexElem = document.createElement("div") + indexElem.innerHtml = resp + # Add items into the DB/contents + for href in indexElem.getElementsByClass("reference"): + # Make links be relative to project root instead of current page + href.setAttr("href", cstring(rootURL & $href.getAttribute("href"))) + db &= href + contents &= href.getAttribute("data-doc-search-tag") + + +var + oldtoc: Element + timer: Timeout + loadIndexFut: Future[void] = nil proc search*() {.exportc.} = proc wrapper() = - let elem = getElementById("searchInput") + let elem = document.getElementById("searchInput") let value = elem.value if value.len != 0: if oldtoc.isNil: - oldtoc = getElementById("tocRoot") + oldtoc = document.getElementById("tocRoot") let results = dosearch(value) replaceById("tocRoot", results) elif not oldtoc.isNil: replaceById("tocRoot", oldtoc) - + # Start loading index as soon as user starts typing. + # Will only be loaded the once anyways + if loadIndexFut == nil: + loadIndexFut = loadIndex() + # Run wrapper once loaded so we don't miss the users query + discard loadIndexFut.then(wrapper) if timer != nil: clearTimeout(timer) timer = setTimeout(wrapper, 400) + +proc copyToClipboard*() {.exportc.} = + {.emit: """ + + function updatePreTags() { + + const allPreTags = document.querySelectorAll("pre:not(.line-nums)") + + allPreTags.forEach((e) => { + + const div = document.createElement("div") + div.classList.add("copyToClipBoard") + + const preTag = document.createElement("pre") + preTag.innerHTML = e.innerHTML + + const button = document.createElement("button") + button.value = e.textContent.replace('...', '') + button.classList.add("copyToClipBoardBtn") + button.style.cursor = "pointer" + + div.appendChild(preTag) + div.appendChild(button) + + e.outerHTML = div.outerHTML + + }) + } + + + function copyTextToClipboard(e) { + const clipBoardContent = e.target.value + navigator.clipboard.writeText(clipBoardContent).then(function() { + e.target.style.setProperty("--clipboard-image", "var(--clipboard-image-selected)") + }, function(err) { + console.error("Could not copy text: ", err); + }); + } + + window.addEventListener("click", (e) => { + if (e.target.classList.contains("copyToClipBoardBtn")) { + copyTextToClipboard(e) + } + }) + + window.addEventListener("mouseover", (e) => { + if (e.target.nodeName === "PRE") { + e.target.nextElementSibling.style.setProperty("--clipboard-image", "var(--clipboard-image-normal)") + } + }) + + window.addEventListener("DOMContentLoaded", updatePreTags) + + """ + .} + +copyToClipboard() +window.addEventListener("DOMContentLoaded", onDOMLoaded) diff --git a/tools/finish.nim b/tools/finish.nim index 4b5131045..69838829c 100644 --- a/tools/finish.nim +++ b/tools/finish.nim @@ -41,7 +41,7 @@ proc downloadMingw(): DownloadResult = let curl = findExe"curl" var cmd: string if curl.len > 0: - cmd = quoteShell(curl) & " --out " & "dist" / mingw & " " & url + cmd = quoteShell(curl) & " --output " & "dist" / mingw & " " & url elif fileExists"bin/nimgrab.exe": cmd = r"bin\nimgrab.exe " & url & " dist" / mingw if cmd.len > 0: diff --git a/tools/grammar_nanny.nim b/tools/grammar_nanny.nim index 397041559..cbdc51efc 100644 --- a/tools/grammar_nanny.nim +++ b/tools/grammar_nanny.nim @@ -3,8 +3,11 @@ import std / [strutils, sets] +when defined(nimPreviewSlimSystem): + import std/syncio + import ".." / compiler / [ - llstream, ast, lexer, options, msgs, idents, + llstream, lexer, options, msgs, idents, lineinfos, pathutils] proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = @@ -13,12 +16,12 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = var stream = llStreamOpen(data) var declaredSyms = initHashSet[string]() var usedSyms = initHashSet[string]() + usedSyms.incl "module" # 'module' is the start rule. if stream != nil: declaredSyms.incl "section" # special case for 'section(RULE)' in the grammar var L: Lexer tok: Token - initToken(tok) openLexer(L, f, stream, cache, config) # load the first token: rawGetTok(L, tok) @@ -35,6 +38,10 @@ proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) = usedSyms.incl word else: rawGetTok(L, tok) + for u in declaredSyms: + if u notin usedSyms: + echo "Unused non-terminal: ", u + for u in usedSyms: if u notin declaredSyms: echo "Undeclared non-terminal: ", u diff --git a/tools/heapdumprepl.nim b/tools/heapdumprepl.nim index 62454165d..4f06cf111 100644 --- a/tools/heapdumprepl.nim +++ b/tools/heapdumprepl.nim @@ -1,4 +1,3 @@ - include std/prelude import intsets @@ -17,7 +16,6 @@ type roots: Table[int, NodeKind] proc add(father: Node; son: int) = - if father.kids.isNil: father.kids = @[] father.kids.add(son) proc renderNode(g: Graph; id: int) = @@ -141,8 +139,8 @@ proc importData(input: string): Graph = close(i) else: quit "error: cannot open " & input - shallowCopy(result.nodes, nodes) - shallowCopy(result.roots, roots) + result.nodes = move nodes + result.roots = move roots if paramCount() == 1: repl(importData(paramStr(1))) diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index 621dc643f..477fb29fa 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -1,14 +1,23 @@ ## Part of 'koch' responsible for the documentation generation. -import os, strutils, osproc, sets, pathnorm, pegs, sequtils +import std/[os, strutils, osproc, sets, pathnorm, sequtils, pegs] + +import officialpackages +export exec + +when defined(nimPreviewSlimSystem): + import std/assertions + from std/private/globs import nativeToUnixPath, walkDirRecFilter, PathEntry import "../compiler/nimpaths" const gaCode* = " --doc.googleAnalytics:UA-48159761-1" + paCode* = " --doc.plausibleAnalytics:nim-lang.org" # errormax: subsequent errors are probably consequences of 1st one; a simple # bug could cause unlimited number of errors otherwise, hard to debug in CI. - nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$#" % system.NimVersion + docDefines = "-d:nimExperimentalLinenoiseExtra" + nimArgs = "--errormax:3 --hint:Conf:off --hint:Path:off --hint:Processing:off --hint:XDeclaredButNotUsed:off --warning:UnusedImport:off -d:boot --putenv:nimversion=$# $#" % [system.NimVersion, docDefines] gitUrl = "https://github.com/nim-lang/Nim" docHtmlOutput = "doc/html" webUploadOutput = "web/upload" @@ -39,38 +48,19 @@ proc findNimImpl*(): tuple[path: string, ok: bool] = proc findNim*(): string = findNimImpl().path -proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = - let prevPath = getEnv("PATH") - if additionalPath.len > 0: - var absolute = additionalPath - if not absolute.isAbsolute: - absolute = getCurrentDir() / absolute - echo("Adding to $PATH: ", absolute) - putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & absolute) - echo(cmd) - if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) - putEnv("PATH", prevPath) - template inFold*(desc, body) = - if existsEnv("TRAVIS"): - echo "travis_fold:start:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::group::" & desc elif existsEnv("TF_BUILD"): echo "##[group]" & desc - body - - if existsEnv("TRAVIS"): - echo "travis_fold:end:" & desc.replace(" ", "_") - elif existsEnv("GITHUB_ACTIONS"): + if existsEnv("GITHUB_ACTIONS"): echo "::endgroup::" elif existsEnv("TF_BUILD"): echo "##[endgroup]" proc execFold*(desc, cmd: string, errorcode: int = QuitFailure, additionalPath = "") = ## Execute shell command. Add log folding for various CI services. - # https://github.com/travis-ci/travis-ci/issues/2285#issuecomment-42724719 let desc = if desc.len == 0: cmd else: desc inFold(desc): exec(cmd, errorcode, additionalPath) @@ -97,44 +87,44 @@ proc nimCompile*(input: string, outputDir = "bin", mode = "c", options = "") = let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input exec cmd -proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "") = - let output = outputDir / input.splitFile.name.exe +proc nimCompileFold*(desc, input: string, outputDir = "bin", mode = "c", options = "", outputName = "") = + let outputName2 = if outputName.len == 0: input.splitFile.name.exe else: outputName.exe + let output = outputDir / outputName2 let cmd = findNim().quoteShell() & " " & mode & " -o:" & output & " " & options & " " & input execFold(desc, cmd) -proc getRst2html(): seq[string] = +const officialPackagesMarkdown = """ +pkgs/atlas/doc/atlas.md +""".splitWhitespace() + +proc getMd2html(): seq[string] = for a in walkDirRecFilter("doc"): let path = a.path - if a.kind == pcFile and path.splitFile.ext == ".rst" and path.lastPathPart notin - ["docs.rst", "nimfix.rst"]: - # maybe we should still show nimfix, could help reviving it + if a.kind == pcFile and path.splitFile.ext == ".md" and path.lastPathPart notin + ["docs.md", + "docstyle.md" # docstyle.md shouldn't be converted to html separately; + # it's included in contributing.md. + ]: # `docs` is redundant with `overview`, might as well remove that file? result.add path - doAssert "doc/manual/var_t_return.rst".unixToNativePath in result # sanity check + for md in officialPackagesMarkdown: + result.add md + doAssert "doc/manual/var_t_return.md".unixToNativePath in result # sanity check const - rstPdfList = """ -manual.rst -lib.rst -tut1.rst -tut2.rst -tut3.rst -nimc.rst -niminst.rst -gc.rst + mdPdfList = """ +manual.md +lib.md +tut1.md +tut2.md +tut3.md +nimc.md +niminst.md +mm.md """.splitWhitespace().mapIt("doc" / it) - doc0 = """ -lib/system/threads.nim -lib/system/channels_builtin.nim -""".splitWhitespace() # ran by `nim doc0` instead of `nim doc` - withoutIndex = """ -lib/wrappers/mysql.nim -lib/wrappers/sqlite3.nim -lib/wrappers/postgres.nim lib/wrappers/tinyc.nim -lib/wrappers/odbcsql.nim lib/wrappers/pcre.nim lib/wrappers/openssl.nim lib/posix/posix.nim @@ -162,6 +152,34 @@ lib/posix/posix_other_consts.nim lib/posix/posix_freertos_consts.nim lib/posix/posix_openbsd_amd64.nim lib/posix/posix_haiku.nim +lib/pure/md5.nim +lib/std/sha1.nim +lib/pure/htmlparser.nim +""".splitWhitespace() + + officialPackagesList = """ +pkgs/asyncftpclient/src/asyncftpclient.nim +pkgs/smtp/src/smtp.nim +pkgs/punycode/src/punycode.nim +pkgs/db_connector/src/db_connector/db_common.nim +pkgs/db_connector/src/db_connector/db_mysql.nim +pkgs/db_connector/src/db_connector/db_odbc.nim +pkgs/db_connector/src/db_connector/db_postgres.nim +pkgs/db_connector/src/db_connector/db_sqlite.nim +pkgs/checksums/src/checksums/md5.nim +pkgs/checksums/src/checksums/sha1.nim +pkgs/checksums/src/checksums/sha2.nim +pkgs/checksums/src/checksums/sha3.nim +pkgs/checksums/src/checksums/bcrypt.nim +pkgs/htmlparser/src/htmlparser.nim +""".splitWhitespace() + + officialPackagesListWithoutIndex = """ +pkgs/db_connector/src/db_connector/mysql.nim +pkgs/db_connector/src/db_connector/sqlite3.nim +pkgs/db_connector/src/db_connector/postgres.nim +pkgs/db_connector/src/db_connector/odbcsql.nim +pkgs/db_connector/src/db_connector/private/dbutils.nim """.splitWhitespace() when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): @@ -172,19 +190,19 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo): result = path.len > 0 and not ret.startsWith ".." proc getDocList(): seq[string] = + ## var docIgnore: HashSet[string] - for a in doc0: docIgnore.incl a for a in withoutIndex: docIgnore.incl a for a in ignoredModules: docIgnore.incl a # don't ignore these even though in lib/system (not include files) const goodSystem = """ -lib/system/io.nim lib/system/nimscript.nim lib/system/assertions.nim lib/system/iterators.nim +lib/system/exceptions.nim lib/system/dollars.nim -lib/system/widestrs.nim +lib/system/ctypes.nim """.splitWhitespace() proc follow(a: PathEntry): bool = @@ -224,49 +242,48 @@ proc buildDocSamples(nimArgs, destPath: string) = exec(findNim().quoteShell() & " doc $# -o:$# $#" % [nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) -proc buildDocPackages(nimArgs, destPath: string) = +proc buildDocPackages(nimArgs, destPath: string, indexOnly: bool) = # compiler docs; later, other packages (perhaps tools, testament etc) let nim = findNim().quoteShell() # to avoid broken links to manual from compiler dir, but a multi-package # structure could be supported later proc docProject(outdir, options, mainproj: string) = - exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $options $mainproj" % [ + exec("$nim doc --project --outdir:$outdir $nimArgs --git.url:$gitUrl $index $options $mainproj" % [ "nim", nim, "outdir", outdir, "nimArgs", nimArgs, "gitUrl", gitUrl, "options", options, "mainproj", mainproj, + "index", if indexOnly: "--index:only" else: "" ]) let extra = "-u:boot" # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") -proc buildDoc(nimArgs, destPath: string) = +proc buildDoc(nimArgs, destPath: string, indexOnly: bool) = # call nim for the documentation: - let rst2html = getRst2html() + let rst2html = getMd2html() var - commands = newSeq[string](rst2html.len + len(doc0) + len(doc) + withoutIndex.len) + commands = newSeq[string](rst2html.len + len(doc) + withoutIndex.len + + officialPackagesList.len + officialPackagesListWithoutIndex.len) i = 0 let nim = findNim().quoteShell() + + let index = if indexOnly: "--index:only" else: "" for d in items(rst2html): - commands[i] = nim & " rst2html $# --git.url:$# -o:$# --index:on $#" % - [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(doc0): - commands[i] = nim & " doc0 $# --git.url:$# -o:$# --index:on $#" % + commands[i] = nim & " md2html $# --git.url:$# -o:$# $# $#" % [nimArgs, gitUrl, - destPath / changeFileExt(splitFile(d).name, "html"), d] + destPath / changeFileExt(splitFile(d).name, "html"), index, d] i.inc for d in items(doc): let extra = if isJsOnly(d): "--backend:js" else: "" var nimArgs2 = nimArgs if d.isRelativeTo("compiler"): doAssert false - commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# --index:on $#" % - [extra, nimArgs2, gitUrl, destPath, d] + commands[i] = nim & " doc $# $# --git.url:$# --outdir:$# $# $#" % + [extra, nimArgs2, gitUrl, destPath, index, d] i.inc for d in items(withoutIndex): commands[i] = nim & " doc $# --git.url:$# -o:$# $#" % @@ -274,37 +291,51 @@ proc buildDoc(nimArgs, destPath: string) = destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc + + for d in items(officialPackagesList): + var nimArgs2 = nimArgs + if d.isRelativeTo("compiler"): doAssert false + commands[i] = nim & " doc $# --outdir:$# --index:on $#" % + [nimArgs2, destPath, d] + i.inc + for d in items(officialPackagesListWithoutIndex): + commands[i] = nim & " doc $# -o:$# $#" % + [nimArgs, + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + mexec(commands) - exec(nim & " buildIndex -o:$1/theindex.html $1" % [destPath]) - # caveat: this works so long it's called before `buildDocPackages` which - # populates `compiler/` with unrelated idx files that shouldn't be in index, - # so should work in CI but you may need to remove your generated html files - # locally after calling `./koch docs`. The clean fix would be for `idx` files - # to be transient with `--project` (eg all in memory). proc nim2pdf(src: string, dst: string, nimArgs: string) = # xxx expose as a `nim` command or in some other reusable way. - let outDir = "build" / "pdflatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 + let outDir = "build" / "xelatextmp" # xxx factor pending https://github.com/timotheecour/Nim/issues/616 # note: this will generate temporary files in gitignored `outDir`: aux toc log out tex - exec("$# rst2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) + exec("$# md2tex $# --outdir:$# $#" % [findNim().quoteShell(), nimArgs, outDir.quoteShell, src.quoteShell]) let texFile = outDir / src.lastPathPart.changeFileExt("tex") - for i in 0..<2: # call LaTeX twice to get cross references right: - let pdflatexLog = outDir / "pdflatex.log" + for i in 0..<3: # call LaTeX three times to get cross references right: + let xelatexLog = outDir / "xelatex.log" # `>` should work on windows, if not, we can use `execCmdEx` - let cmd = "pdflatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, pdflatexLog.quoteShell] - exec(cmd) # on error, user can inspect `pdflatexLog` + let cmd = "xelatex -interaction=nonstopmode -output-directory=$# $# > $#" % [outDir.quoteShell, texFile.quoteShell, xelatexLog.quoteShell] + exec(cmd) # on error, user can inspect `xelatexLog` + if i == 1: # build .ind file + var texFileBase = texFile + texFileBase.removeSuffix(".tex") + let cmd = "makeindex $# > $#" % [ + texFileBase.quoteShell, xelatexLog.quoteShell] + exec(cmd) moveFile(texFile.changeFileExt("pdf"), dst) -proc buildPdfDoc*(nimArgs, destPath: string) = +proc buildPdfDoc*(args: string, destPath: string) = + let args = nimArgs & " " & args var pdfList: seq[string] createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - doAssert false, "pdflatex not found" # or, raise an exception + if os.execShellCmd("xelatex -version") != 0: + doAssert false, "xelatex not found" # or, raise an exception else: - for src in items(rstPdfList): + for src in items(mdPdfList): let dst = destPath / src.lastPathPart.changeFileExt("pdf") pdfList.add dst - nim2pdf(src, dst, nimArgs) + nim2pdf(src, dst, args) echo "\nOutput PDF files: \n ", pdfList.join(" ") # because `nim2pdf` is a bit verbose proc buildJS(): string = @@ -317,11 +348,26 @@ proc buildJS(): string = proc buildDocsDir*(args: string, dir: string) = let args = nimArgs & " " & args let docHackJsSource = buildJS() + gitClonePackages(@["asyncftpclient", "punycode", "smtp", "db_connector", "checksums", "atlas", "htmlparser"]) createDir(dir) buildDocSamples(args, dir) - buildDoc(args, dir) # bottleneck + + # generate `.idx` files and top-level `theindex.html`: + buildDoc(args, dir, indexOnly=true) # bottleneck + let nim = findNim().quoteShell() + exec(nim & " buildIndex -o:$1/theindex.html $1" % [dir]) + # caveat: this works so long it's called before `buildDocPackages` which + # populates `compiler/` with unrelated idx files that shouldn't be in index, + # so should work in CI but you may need to remove your generated html files + # locally after calling `./koch docs`. The clean fix would be for `idx` files + # to be transient with `--project` (eg all in memory). + buildDocPackages(args, dir, indexOnly=true) + + # generate HTML and package-level `theindex.html`: + buildDoc(args, dir, indexOnly=false) # bottleneck + buildDocPackages(args, dir, indexOnly=false) + copyFile(dir / "overview.html", dir / "index.html") - buildDocPackages(args, dir) copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart) proc buildDocs*(args: string, localOnly = false, localOutDir = "") = diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index f7aeb72c6..1c3670fd9 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -1,148 +1,120 @@ #compdef nim +# Installation note: +# Please name this file as _nim (with underscore!) and copy it to a +# completions directory, either: +# - system wide one, like /usr/share/zsh/functions/Completion/Unix/ on Linux +# - or to a user directory like ~/scripts. Then you also need to add +# that directory in your ~/.zshrc to `fpath` array like so: +# fpath=( ~/scripts "${fpath[@]}" ) + _nim() { - _arguments -C \ - ':command:(( - {compile,c}\:compile\ project\ with\ default\ code\ generator\ C - doc\:generate\ the\ documentation\ for\ inputfile - {compileToC,cc}\:compile\ project\ with\ C\ code\ generator - {compileToCpp,cpp}\:compile\ project\ to\ C++\ code - {compileToOC,objc}\:compile\ project\ to\ Objective\ C\ code - js\:compile\ project\ to\ Javascript - e\:run\ a\ Nimscript\ file - rst2html\:convert\ a\ reStructuredText\ file\ to\ HTML - rst2tex\:convert\ a\ reStructuredText\ file\ to\ TeX - jsondoc\:extract\ the\ documentation\ to\ a\ json\ file - buildIndex\:build\ an\ index\ for\ the\ whole\ documentation - genDepend\:generate\ a\ DOT\ file\ containing\ the\ module\ dependency\ graph - dump\:dump\ all\ defined\ conditionals\ and\ search\ paths - check\:checks\ the\ project\ for\ syntax\ and\ semantic - ))' \ - '*-r[run the application]' \ - '*--run[run the application]' \ - '*-p=[add path to search paths]' \ - '*--path=[add path to search paths]' \ - '*-d=[define a conditional symbol]' \ - '*--define=[define a conditional symbol]' \ - '*-u=[undefine a conditional symbol]' \ - '*--undef=[undefine a conditional symbol]' \ - '*-f[force rebuilding of all modules]' \ - '*--forceBuild[force rebuilding of all modules]' \ - '*--stackTrace=on[turn stack tracing on]' \ - '*--stackTrace=off[turn stack tracing off]' \ - '*--lineTrace=on[turn line tracing on]' \ - '*--lineTrace=off[turn line tracing off]' \ - '*--threads=on[turn support for multi-threading on]' \ - '*--threads=off[turn support for multi-threading off]' \ - '*-x=on[turn all runtime checks on]' \ - '*-x=off[turn all runtime checks off]' \ - '*--checks=on[turn all runtime checks on]' \ - '*--checks=off[turn all runtime checks off]' \ - '*--objChecks=on[turn obj conversion checks on]' \ - '*--objChecks=off[turn obj conversion checks off]' \ - '*--fieldChecks=on[turn case variant field checks on]' \ - '*--fieldChecks=off[turn case variant field checks off]' \ - '*--rangeChecks=on[turn range checks on]' \ - '*--rangeChecks=off[turn range checks off]' \ - '*--boundChecks=on[turn bound checks on]' \ - '*--boundChecks=off[turn bound checks off]' \ - '*--overflowChecks=on[turn int over-/underflow checks on]' \ - '*--overflowChecks=off[turn int over-/underflow checks off]' \ - '*-a[turn assertions on]' \ - '*-a[turn assertions off]' \ - '*--assertions=on[turn assertions on]' \ - '*--assertions=off[turn assertions off]' \ - '*--floatChecks=on[turn all floating point (NaN/Inf) checks on]' \ - '*--floatChecks=off[turn all floating point (NaN/Inf) checks off]' \ - '*--nanChecks=on[turn NaN checks on]' \ - '*--nanChecks=off[turn NaN checks off]' \ - '*--infChecks=on[turn Inf checks on]' \ - '*--infChecks=off[turn Inf checks off]' \ - '*--nilChecks=on[turn nil checks on]' \ - '*--nilChecks=off[turn nil checks off]' \ - '*--opt=none[do not optimize]' \ - '*--opt=speed[optimize for speed|size - use -d:release for a release build]' \ - '*--opt=size[optimize for size]' \ - '*--debugger:native[use native debugger (gdb)]' \ - '*--app=console[generate a console app]' \ - '*--app=gui[generate a GUI app]' \ - '*--app=lib[generate a dynamic library]' \ - '*--app=staticlib[generate a static library]' \ - '*--cpu=alpha[compile for Alpha architecture]' \ - '*--cpu=amd64[compile for x86_64 architecture]' \ - '*--cpu=arm[compile for ARM architecture]' \ - '*--cpu=arm64[compile for ARM64 architecture]' \ - '*--cpu=avr[compile for AVR architecture]' \ - '*--cpu=esp[compile for ESP architecture]' \ - '*--cpu=hppa[compile for HPPA architecture]' \ - '*--cpu=i386[compile for i386 architecture]' \ - '*--cpu=ia64[compile for ia64 architecture]' \ - '*--cpu=js[compile to JavaScript]' \ - '*--cpu=m68k[compile for m68k architecture]' \ - '*--cpu=mips[compile for MIPS architecture]' \ - '*--cpu=mipsel[compile for MIPS EL architecture]' \ - '*--cpu=mips64[compile for MIPS64 architecture]' \ - '*--cpu=mips64el[compile for MIPS64 EL architecture]' \ - '*--cpu=msp430[compile for msp430 architecture]' \ - '*--cpu=nimvm[compile for Nim VM]' \ - '*--cpu=powerpc[compile for PowerPC architecture]' \ - '*--cpu=powerpc64[compile for PowerPC64 architecture]' \ - '*--cpu=powerpc64el[compile for PowerPC64 EL architecture]' \ - '*--cpu=riscv32[compile for RISC-V 32 architecture]' \ - '*--cpu=riscv64[compile for RISC-V 64 architecture]' \ - '*--cpu=sparc[compile for SPARC architecture]' \ - '*--cpu=sparc64[compile for SPARC64 architecture]' \ - '*--cpu=vm[compile for Nim VM]' \ - '*--cpu=wasm32[compile to WASM 32]' \ - '*--gc=refc[use reference counting garbage collection]' \ - '*--gc=arc[use ARC garbage collection]' \ - '*--gc=orc[use ORC garbage collection]' \ - '*--gc=markAndSweep[use mark-and-sweep garbage collection]' \ - '*--gc=boehm[use Boehm garbage collection]' \ - '*--gc=go[use Go garbage collection]' \ - '*--gc=regions[use region-based memory management]' \ - '*--gc=none[disable garbage collection]' \ - '*--os=Standalone[generate a stand-alone executable]' \ - '*--os=AIX[compile for AIX]' \ - '*--os=Amiga[compile for Amiga OS]' \ - '*--os=Android[compile for Android]' \ - '*--os=Any[compile for any OS]' \ - '*--os=Atari[compile for Atari]' \ - '*--os=DOS[compile for DOS]' \ - '*--os=DragonFly[compile for DragonFly]' \ - '*--os=FreeBSD[compile for FreeBSD]' \ - '*--os=FreeRTOS[compile for FreeRTOS]' \ - '*--os=Genode[compile for Genode]' \ - '*--os=Haiku[compile for Haiku]' \ - '*--os=iOS[compile for iOS]' \ - '*--os=Irix[compile for Irix]' \ - '*--os=Linux[compile for Linux]' \ - '*--os=MacOS[compile for MacOS]' \ - '*--os=MacOSX[compile for MacOSX]' \ - '*--os=MorphOS[compile for MorphOS]' \ - '*--os=NetBSD[compile for NetBSD]' \ - '*--os=Netware[compile for Netware]' \ - '*--os=NimVM[compile for NimVM]' \ - '*--os=NintendoSwitch[compile for NintendoSwitch]' \ - '*--os=OS2[compile for OS2]' \ - '*--os=OpenBSD[compile for OpenBSD]' \ - '*--os=PalmOS[compile for PalmOS]' \ - '*--os=QNX[compile for QNX]' \ - '*--os=SkyOS[compile for SkyOS]' \ - '*--os=Solaris[compile for Solaris]' \ - '*--os=VxWorks[compile for VxWorks]' \ - '*--os=Windows[compile for Windows]' \ - '*--os=JS[generate javascript]' \ - '*--panics=off[turn panics into process terminations: off by default]' \ - '*--panics=on[turn panics into process terminations]' \ - '*--verbosity=0[set verbosity to 0]' \ - '*--verbosity=1[set verbosity to 1 (default)]' \ - '*--verbosity=2[set verbosity to 2]' \ - '*--verbosity=3[set verbosity to 3]' \ - '*--hints=on[print compilation hints]' \ - '*--hints=off[disable compilation hints]' \ - '*--hints=list[print compilation hints list]' \ - ':filename:_files -g"*.nim"' + local -a nimCommands=( + {compile,c}:'compile project with default code generator C' + {compileToC,cc}:'compile project with C code generator' + {compileToCpp,cpp}:'compile project to C++ code' + {compileToOC,objc}:'compile project to Objective C code' + 'js:compile project to Javascript' + 'e:run a Nimscript file' + 'doc:generate the HTML documentation for inputfile' + 'rst2html:convert a reStructuredText file to HTML' + 'doc2tex:generate the documentation for inputfile to LaTeX' + 'rst2tex:convert a reStructuredText file to TeX' + 'jsondoc:extract the documentation to a json file' + 'buildIndex:build an index for the whole documentation' + 'genDepend:generate a DOT file containing the module dependency graph' + 'dump:dump all defined conditionals and search paths' + 'check:checks the project for syntax and semantic' + {--help,-h}:'basic help' + '--fullhelp:show all switches' + {-v,--version}:'show version' + ) + + _arguments '*:: :->anyState' && return 0 + + if (( CURRENT == 1 )); then + _describe -t commands "Nim commands" nimCommands -V1 + return + fi + + local -a conditionalSymbols=( + release danger mingw androidNDK useNimRtl useMalloc noSignalHandler ssl + debug leanCompiler gcDestructors) + local -a sharedOpts=( + {--define\\:-,-d\\:-}'[define a conditional symbol]:x:($conditionalSymbols)' + {--undef\\:-,-u\\:-}'[undefine a conditional symbol]:x:($conditionalSymbols)' + {--path\\:-,-p\\:-}'[add path to search paths]:x:_files' + '--verbosity\:-[set verbosity level (default\: 1)]:x:(0 1 2 3)' + '--hints\:-[print compilation hints? (or `list`)]:x:(on off list)' + ) + local runOpts=( + {--run,-r}'[run the application]' + ) + local docOpts=( + '--index\:-[enable index .idx files?]:x:(on off)' + '--project\:-[output any dependency for doc?]:x:(on off)' + '--docInternal\:-[generate module-private documentation?]:x:(on off)' + ) + local -a codeOpts=( + {--forceBuild,-f}'[force rebuilding of all modules]' + '--stackTrace\:-[enable stack tracing?]:x:(on off)' + '--lineTrace\:-[enable line tracing?]:x:(on off)' + '--threads\:-[enable support for multi-threading?]:x:(on off)' + {--checks\\:-,-x\\:-}'[enable/disable all runtime checks?]:x:(on off)' + '--objChecks\:-[enable obj conversion checks]:x:(on off)' + '--fieldChecks\:-[enable case variant field checks?]:x:(on off)' + '--rangeChecks\:-[enable range checks?]:x:(on off)' + '--boundChecks\:-[enable bound checks?]:x:(on off)' + '--overflowChecks\:-[enable integer over-/underflow checks?]:x:(on off)' + {--assertions\\:-,-a\\:-}'[enable assertions?]:x:(on off)' + '--floatChecks\:-[enable floating point (NaN/Inf) checks?]:x:(on off)' + '--nanChecks\:-[enable NaN checks?]:x:(on off)' + '--infChecks\:-[enable Inf checks?]:x:(on off)' + '--nilChecks\:-[enable nil checks?]:x:(on off)' + '--expandArc\:-[show how given proc looks before final backend pass]' + '--expandMacro\:-[dump every generated AST from given macro]' + ) + local -a nativeOpts=( + '--opt\:-[optimization mode]:x:(none speed size)' + '--debugger\:native[use native debugger (gdb)]' + '--app\:-[generate this type of app (lib=dynamic)]:x:(console gui lib staticlib)' + '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 vm wasm32)' + '--gc\:-[memory management algorithm to use (default\: refc)]:x:(refc arc orc markAndSweep boehm go regions none)' + '--os\:-[operating system to compile for]:x:(AIX Amiga Android Any Atari DOS DragonFly FreeBSD FreeRTOS Genode Haiku iOS Irix JS Linux MacOS MacOSX MorphOS NetBSD Netware NimVM NintendoSwitch OS2 OpenBSD PalmOS Standalone QNX SkyOS Solaris VxWorks Windows)' + '--panics\:-[turn panics into process termination (default\: off)]:x:(off on)' + ) + + case "$words[1]" in + compile|c|compileToC|cpp|compileToCpp|compileToOC|objc) + _arguments $codeOpts $runOpts $sharedOpts $nativeOpts \ + '*:filename:_files -g"*.nim"' + ;; + js) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nim"' + ;; + e) + _arguments $codeOpts $runOpts $sharedOpts \ + '*:filename:_files -g"*.nims"' + ;; + doc|doc2tex|jsondoc) + _arguments $runOpts $sharedOpts '*:filename:_files -g"*.nim"' + ;; + rst2html|rst2tex) + _arguments $runOpts $sharedOpts $docOpts '*:filename:_files -g"*.rst"' + ;; + buildIndex|genDepend|check) + _arguments $sharedOpts '*:filename:_files -g"*.nim"' + ;; + dump) + _arguments $sharedOpts + ;; + *) + _arguments '*:filename:_files -g"*"' + ;; + esac + + return 1 } _nim "$@" diff --git a/tools/nimblepkglist.nim b/tools/nimblepkglist.nim index c4bec4485..92e1cad20 100644 --- a/tools/nimblepkglist.nim +++ b/tools/nimblepkglist.nim @@ -56,16 +56,16 @@ proc processContent(content: string) = var officialPkgListDiv = document.getElementById("officialPkgList") officialPkgListDiv.innerHTML = - p("There are currently " & $officialCount & + (p("There are currently " & $officialCount & " official packages in the Nimble package repository.") & - ul(officialList) + ul(officialList)).cstring var unofficialPkgListDiv = document.getElementById("unofficialPkgList") unofficialPkgListDiv.innerHTML = - p("There are currently " & $unofficialCount & + (p("There are currently " & $unofficialCount & " unofficial packages in the Nimble package repository.") & - ul(unofficialList) + ul(unofficialList)).cstring proc gotPackageList(apiReply: TData) {.exportc.} = let decoded = decodeContent($apiReply.content) @@ -76,5 +76,5 @@ proc gotPackageList(apiReply: TData) {.exportc.} = var unofficialPkgListDiv = document.getElementById("unofficialPkgList") let msg = p("Unable to retrieve package list: ", code(getCurrentExceptionMsg())) - officialPkgListDiv.innerHTML = msg - unofficialPkgListDiv.innerHTML = msg + officialPkgListDiv.innerHTML = msg.cstring + unofficialPkgListDiv.innerHTML = msg.cstring diff --git a/tools/nimfind.nim b/tools/nimfind.nim deleted file mode 100644 index 9c8247606..000000000 --- a/tools/nimfind.nim +++ /dev/null @@ -1,234 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2018 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Nimfind is a tool that helps to give editors IDE like capabilities. - -when not defined(nimcore): - {.error: "nimcore MUST be defined for Nim's core tooling".} -when not defined(nimfind): - {.error: "nimfind MUST be defined for Nim's nimfind tool".} - -const Usage = """ -Nimfind - Tool to find declarations or usages for Nim symbols -Usage: - nimfind [options] file.nim:line:col - -Options: - --help, -h show this help - --rebuild rebuild the index - --project:file.nim use file.nim as the entry point - -In addition, all command line options of Nim that do not affect code generation -are supported. -""" - -import strutils, os, parseopt - -import "../compiler" / [options, commands, modules, sem, - passes, passaux, msgs, ast, - idents, modulegraphs, lineinfos, cmdlinehelper, - pathutils] - -import db_sqlite - -proc createDb(db: DbConn) = - db.exec(sql""" - create table if not exists filenames( - id integer primary key, - fullpath varchar(8000) not null - ); - """) - db.exec sql"create index if not exists FilenameIx on filenames(fullpath);" - - # every sym can have potentially 2 different definitions due to forward - # declarations. - db.exec(sql""" - create table if not exists syms( - id integer primary key, - nimid integer not null, - name varchar(256) not null, - defline integer not null, - defcol integer not null, - deffile integer not null, - deflineB integer not null default 0, - defcolB integer not null default 0, - deffileB integer not null default 0, - foreign key (deffile) references filenames(id), - foreign key (deffileB) references filenames(id) - ); - """) - - db.exec(sql""" - create table if not exists usages( - id integer primary key, - nimid integer not null, - line integer not null, - col integer not null, - colB integer not null, - file integer not null, - foreign key (file) references filenames(id), - foreign key (nimid) references syms(nimid) - ); - """) - db.exec sql"create index if not exists UsagesIx on usages(file, line);" - -proc toDbFileId*(db: DbConn; conf: ConfigRef; fileIdx: FileIndex): int = - if fileIdx == FileIndex(-1): return -1 - let fullpath = toFullPath(conf, fileIdx) - let row = db.getRow(sql"select id from filenames where fullpath = ?", fullpath) - let id = row[0] - if id.len == 0: - result = int db.insertID(sql"insert into filenames(fullpath) values (?)", - fullpath) - else: - result = parseInt(id) - -type - FinderRef = ref object of RootObj - db: DbConn - -proc writeDef(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into syms(nimid, name, defline, defcol, deffile) values (?, ?, ?, ?, ?)""", - s.id, s.name.s, info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc writeDefResolveForward(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""update syms set deflineB = ?, defcolB = ?, deffileB = ? - where nimid = ?""", info.line, info.col, - toDbFileId(f.db, graph.config, info.fileIndex), s.id) - -proc writeUsage(graph: ModuleGraph; s: PSym; info: TLineInfo) = - let f = FinderRef(graph.backend) - f.db.exec(sql"""insert into usages(nimid, line, col, colB, file) values (?, ?, ?, ?, ?)""", - s.id, info.line, info.col, info.col + s.name.s.len - 1, - toDbFileId(f.db, graph.config, info.fileIndex)) - -proc performSearch(conf: ConfigRef; dbfile: AbsoluteFile) = - var db = open(connection=string dbfile, user="nim", password="", - database="nim") - let pos = conf.m.trackPos - let fid = toDbFileId(db, conf, pos.fileIndex) - let known = toFullPath(conf, pos.fileIndex) - let nimids = db.getRow(sql"""select distinct nimid from usages where line = ? and file = ? and ? between col and colB""", - pos.line, fid, pos.col) - if nimids.len > 0: - var idSet = "" - for id in nimids: - if idSet.len > 0: idSet.add ", " - idSet.add id - var outputLater = "" - for r in db.rows(sql"""select line, col, filenames.fullpath from usages - inner join filenames on filenames.id = file - where nimid in (?)""", idSet): - let line = parseInt(r[0]) - let col = parseInt(r[1]) - let file = r[2] - if file == known and line == pos.line.int: - # output the line we already know last: - outputLater.add file & ":" & $line & ":" & $(col+1) & "\n" - else: - echo file, ":", line, ":", col+1 - if outputLater.len > 0: stdout.write outputLater - close(db) - -proc setupDb(g: ModuleGraph; dbfile: AbsoluteFile) = - var f = FinderRef() - removeFile(dbfile) - f.db = open(connection=string dbfile, user="nim", password="", - database="nim") - createDb(f.db) - f.db.exec(sql"pragma journal_mode=off") - # This MUST be turned off, otherwise it's way too slow even for testing purposes: - f.db.exec(sql"pragma SYNCHRONOUS=off") - f.db.exec(sql"pragma LOCKING_MODE=exclusive") - g.backend = f - -proc mainCommand(graph: ModuleGraph) = - let conf = graph.config - let dbfile = getNimcacheDir(conf) / RelativeFile"nimfind.db" - if not fileExists(dbfile) or optForceFullMake in conf.globalOptions: - clearPasses(graph) - registerPass graph, verbosePass - registerPass graph, semPass - conf.setCmd cmdIdeTools - wantMainModule(conf) - setupDb(graph, dbfile) - - graph.onDefinition = writeUsage # writeDef - graph.onDefinitionResolveForward = writeUsage # writeDefResolveForward - graph.onUsage = writeUsage - - if not fileExists(conf.projectFull): - quit "cannot find file: " & conf.projectFull.string - add(conf.searchPaths, conf.libpath) - conf.setErrorMaxHighMaybe - try: - compileProject(graph) - finally: - close(FinderRef(graph.backend).db) - performSearch(conf, dbfile) - -proc processCmdLine*(pass: TCmdLinePass, cmd: string; conf: ConfigRef) = - var p = parseopt.initOptParser(cmd) - while true: - parseopt.next(p) - case p.kind - of cmdEnd: break - of cmdLongOption, cmdShortOption: - case p.key.normalize - of "help", "h": - stdout.writeLine(Usage) - quit() - of "project": - conf.projectName = p.val - of "rebuild": - incl conf.globalOptions, optForceFullMake - else: processSwitch(pass, p, conf) - of cmdArgument: - let info = p.key.split(':') - if info.len == 3: - conf.projectName = findProjectNimFile(conf, info[0].splitFile.dir) - if conf.projectName.len == 0: conf.projectName = info[0] - try: - conf.m.trackPos = newLineInfo(conf, AbsoluteFile info[0], - parseInt(info[1]), parseInt(info[2])-1) - except ValueError: - quit "invalid command line" - else: - quit "invalid command line" - -proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = - let self = NimProg( - suggestMode: true, - processCmdLine: processCmdLine - ) - self.initDefinesProg(conf, "nimfind") - - if paramCount() == 0: - stdout.writeLine(Usage) - return - - self.processCmdLineAndProjectPath(conf) - - # Find Nim's prefix dir. - let binaryPath = findExe("nim") - if binaryPath == "": - raise newException(IOError, - "Cannot find Nim standard library: Nim compiler not in PATH") - conf.prefixDir = AbsoluteDir binaryPath.splitPath().head.parentDir() - if not dirExists(conf.prefixDir / RelativeDir"lib"): - conf.prefixDir = AbsoluteDir"" - - var graph = newModuleGraph(cache, conf) - if self.loadConfigsAndProcessCmdLine(cache, conf, graph): - mainCommand(graph) - -handleCmdLine(newIdentCache(), newConfigRef()) diff --git a/tools/nimfind.nims b/tools/nimfind.nims deleted file mode 100644 index 3013c360d..000000000 --- a/tools/nimfind.nims +++ /dev/null @@ -1,2 +0,0 @@ ---define:nimfind ---define:nimcore diff --git a/tools/nimgrab.nim b/tools/nimgrab.nim index ee5eced1e..c86159739 100644 --- a/tools/nimgrab.nim +++ b/tools/nimgrab.nim @@ -1,33 +1,21 @@ - - -when defined(windows): - import os, urldownloader - - proc syncDownload(url, file: string) = - proc progress(status: DownloadStatus, progress: uint, total: uint, - message: string) {.gcsafe.} = - echo "Downloading " & url - let t = total.BiggestInt - if t != 0: - echo clamp(int(progress.BiggestInt*100 div t), 0, 100), "%" - else: - echo "0%" - - downloadToFile(url, file, {optUseCache}, progress) - echo "100%" - -else: - import os, asyncdispatch, httpclient - - proc syncDownload(url, file: string) = - var client = newHttpClient() - proc onProgressChanged(total, progress, speed: BiggestInt) = - echo "Downloading " & url & " " & $(speed div 1000) & "kb/s" - echo clamp(int(progress*100 div total), 0, 100), "%" - - client.onProgressChanged = onProgressChanged - client.downloadFile(url, file) - echo "100%" +import std/[os, httpclient] + +proc syncDownload(url, file: string) = + let client = newHttpClient() + proc onProgressChanged(total, progress, speed: BiggestInt) = + var message = "Downloading " + message.add url + message.add ' ' + message.addInt speed div 1000 + message.add "kb/s\n" + message.add $clamp(int(progress * 100 div total), 0, 100) + message.add '%' + echo message + + client.onProgressChanged = onProgressChanged + client.downloadFile(url, file) + client.close() + echo "100%" if os.paramCount() != 2: quit "Usage: nimgrab <url> <file>" diff --git a/tools/nimgrep.nim b/tools/nimgrep.nim index cb46f30b8..599c616ba 100644 --- a/tools/nimgrep.nim +++ b/tools/nimgrep.nim @@ -11,111 +11,12 @@ import os, strutils, parseopt, pegs, re, terminal, osproc, tables, algorithm, times const - Version = "1.6.0" + Version = "2.0.0" Usage = "nimgrep - Nim Grep Searching and Replacement Utility Version " & Version & """ (c) 2012-2020 Andreas Rumpf - -Usage: -* To search: - nimgrep [options] PATTERN [(FILE/DIRECTORY)*/-] -* To replace: - nimgrep [options] PATTERN --replace REPLACEMENT (FILE/DIRECTORY)*/- -* To list file names: - nimgrep [options] --filenames [PATTERN] [(FILE/DIRECTORY)*] - -Positional arguments, from left to right: -* PATERN is either Regex (default) or Peg if --peg is specified. - PATTERN and REPLACEMENT should be skipped when --stdin is specified. -* REPLACEMENT supports $1, $# notations for captured groups in PATTERN. - Note: --replace mode DOES NOT ask confirmation unless --confirm is specified! -* Final arguments are a list of paths (FILE/DIRECTORY) or a standalone - minus '-' (pipe) or not specified (empty). Note for the empty case: when - no FILE/DIRECTORY/- is specified nimgrep DOES NOT read the pipe, but - searches files in the current dir instead! - - read buffer once from stdin: pipe or terminal input; - in --replace mode the result is directed to stdout; - it's not compatible with --stdin, --filenames, --confirm - (empty) current directory '.' is assumed (not with --replace) - For any given DIRECTORY nimgrep searches only its immediate files without - traversing sub-directories unless --recursive is specified. - In replacement mode all 3 positional arguments are required to avoid damaging. - -Options: -* Mode of operation: - --find, -f find the PATTERN (default) - --replace, -! replace the PATTERN to REPLACEMENT, rewriting the files - --confirm confirm each occurrence/replacement; there is a chance - to abort any time without touching the file - --filenames just list filenames. Provide a PATTERN to find it in - the filenames (not in the contents of a file) or run - with empty pattern to just list all files: - nimgrep --filenames # In current directory - nimgrep --filenames "" DIRECTORY # Note empty pattern "" - -* Interprete patterns: - --peg PATTERN and PAT are Peg - --re PATTERN and PAT are regular expressions (default) - --rex, -x use the "extended" syntax for the regular expression - so that whitespace is not significant - --word, -w matches should have word boundaries (buggy for pegs!) - --ignoreCase, -i be case insensitive in PATTERN and PAT - --ignoreStyle, -y be style insensitive in PATTERN and PAT - NOTE: PATERN and patterns PAT (see below in other options) are all either - Regex or Peg simultaneously and options --rex, --word, --ignoreCase, - --ignoreStyle are applied to all of them. - -* File system walk: - --recursive, -r process directories recursively - --follow follow all symlinks when processing recursively - --ext:EX1|EX2|... only search the files with the given extension(s), - empty one ("--ext") means files with missing extension - --noExt:EX1|... exclude files having given extension(s), use empty one to - skip files with no extension (like some binary files are) - --includeFile:PAT search only files whose names contain pattern PAT - --excludeFile:PAT skip files whose names contain pattern PAT - --includeDir:PAT search only files with whole directory path containing PAT - --excludeDir:PAT skip directories whose name (not path) contain pattern PAT - --if,--ef,--id,--ed abbreviations of 4 options above - --sortTime order files by the last modification time (default: off): - -s[:asc|desc] ascending (recent files go last) or descending - -* Filter file content: - --match:PAT select files containing a (not displayed) match of PAT - --noMatch:PAT select files not containing any match of PAT - --bin:on|off|only process binary files? (detected by \0 in first 1K bytes) - (default: on - binary and text files treated the same way) - --text, -t process only text files, the same as --bin:off - -* Represent results: - --nocolor output will be given without any colors - --color[:on] force color even if output is redirected (default: auto) - --colorTheme:THEME select color THEME from 'simple' (default), - 'bnw' (black and white) ,'ack', or 'gnu' (GNU grep) - --count only print counts of matches for files that matched - --context:N, -c:N print N lines of leading context before every match and - N lines of trailing context after it (default N: 0) - --afterContext:N, - -a:N print N lines of trailing context after every match - --beforeContext:N, - -b:N print N lines of leading context before every match - --group, -g group matches by file - --newLine, -l display every matching line starting from a new line - --cols[:N] limit max displayed columns/width of output lines from - files by N characters, cropping overflows (default: off) - --cols:auto, -% calculate columns from terminal width for every line - --onlyAscii, -@ display only printable ASCII Latin characters 0x20-0x7E - substitutions: 0 -> ^@, 1 -> ^A, ... 0x1F -> ^_, - 0x7F -> '7F, ..., 0xFF -> 'FF -* Miscellaneous: - --threads:N, -j:N speed up search by N additional workers (default: 0, off) - --stdin read PATTERN from stdin (to avoid the shell's confusing - quoting rules) and, if --replace given, REPLACEMENT - --verbose be verbose: list every processed file - --help, -h shows this help - --version, -v shows the version -""" +""" & slurp "../doc/nimgrep_cmdline.txt" # Limitations / ideas / TODO: # * No unicode support with --cols @@ -194,26 +95,34 @@ type filename: string, fileResult: FileResult] WalkOpt = tuple # used for walking directories/producing paths extensions: seq[string] - skipExtensions: seq[string] - excludeFile: seq[string] - includeFile: seq[string] - includeDir : seq[string] - excludeDir : seq[string] + notExtensions: seq[string] + filename: seq[string] + notFilename: seq[string] + dirPath: seq[string] + notDirPath: seq[string] + dirname : seq[string] + notDirname : seq[string] WalkOptComp[Pat] = tuple # a compiled version of the previous - excludeFile: seq[Pat] - includeFile: seq[Pat] - includeDir : seq[Pat] - excludeDir : seq[Pat] + filename: seq[Pat] + notFilename: seq[Pat] + dirname : seq[Pat] + notDirname : seq[Pat] + dirPath: seq[Pat] + notDirPath: seq[Pat] SearchOpt = tuple # used for searching inside a file - patternSet: bool # to distinguish uninitialized 'pattern' and empty one - pattern: string # main PATTERN - checkMatch: string # --match - checkNoMatch: string # --nomatch - checkBin: Bin # --bin + patternSet: bool # To distinguish uninitialized/empty 'pattern' + pattern: string # Main PATTERN + inFile: seq[string] # --inFile, --inf + notInFile: seq[string] # --notinFile, --ninf + inContext: seq[string] # --inContext, --inc + notInContext: seq[string] # --notinContext, --ninc + checkBin: Bin # --bin, --text SearchOptComp[Pat] = tuple # a compiled version of the previous pattern: Pat - checkMatch: Pat - checkNoMatch: Pat + inFile: seq[Pat] + notInFile: seq[Pat] + inContext: seq[Pat] + notInContext: seq[Pat] SinglePattern[PAT] = tuple # compile single pattern for replacef pattern: PAT Column = tuple # current column info for the cropping (--limit) feature @@ -748,7 +657,7 @@ template updateCounters(output: Output) = proc printInfo(filename:string, output: Output) = case output.kind of openError: - printError("can not open path " & filename & " " & output.msg) + printError("cannot open path '" & filename & "': " & output.msg) of rejected: if optVerbose in options: echo "(rejected: ", output.reason, ")" @@ -818,6 +727,9 @@ iterator searchFile(pattern: Pattern; buffer: string): Output = pre: pre, match: move(curMi)) i = t.last+1 + when typeof(pattern) is Regex: + if buffer.len > MaxReBufSize: + yield Output(kind: openError, msg: "PCRE size limit is " & $MaxReBufSize) func detectBin(buffer: string): bool = for i in 0 ..< min(1024, buffer.len): @@ -903,6 +815,33 @@ template declareCompiledPatterns(compiledStruct: untyped, body {.hint[XDeclaredButNotUsed]: on.} +template ensureIncluded(includePat: seq[Pattern], str: string, + body: untyped) = + if includePat.len != 0: + var matched = false + for pat in includePat: + if str.contains(pat): + matched = true + break + if not matched: + body + +template ensureExcluded(excludePat: seq[Pattern], str: string, + body: untyped) = + {.warning[UnreachableCode]: off.} + for pat in excludePat: + if str.contains(pat, 0): + body + break + {.warning[UnreachableCode]: on.} + +func checkContext(context: string, searchOptC: SearchOptComp[Pattern]): bool = + ensureIncluded searchOptC.inContext, context: + return false + ensureExcluded searchOptC.notInContext, context: + return false + result = true + iterator processFile(searchOptC: SearchOptComp[Pattern], filename: string, yieldContents=false): Output = var buffer: string @@ -932,13 +871,13 @@ iterator processFile(searchOptC: SearchOptComp[Pattern], filename: string, reason = "text file" if not reject: - if searchOpt.checkMatch != "": - reject = not contains(buffer, searchOptC.checkMatch, 0) + ensureIncluded searchOptC.inFile, buffer: + reject = true reason = "doesn't contain a requested match" if not reject: - if searchOpt.checkNoMatch != "": - reject = contains(buffer, searchOptC.checkNoMatch, 0) + ensureExcluded searchOptC.notInFile, buffer: + reject = true reason = "contains a forbidden match" if reject: @@ -948,20 +887,50 @@ iterator processFile(searchOptC: SearchOptComp[Pattern], filename: string, else: var found = false var cnt = 0 - for output in searchFile(searchOptC.pattern, buffer): - found = true - if optCount notin options: - yield output - else: - if output.kind in {blockFirstMatch, blockNextMatch}: - inc(cnt) + let skipCheckContext = (searchOpt.notInContext.len == 0 and + searchOpt.inContext.len == 0) + if skipCheckContext: + for output in searchFile(searchOptC.pattern, buffer): + found = true + if optCount notin options: + yield output + else: + if output.kind in {blockFirstMatch, blockNextMatch}: + inc(cnt) + else: + var context: string + var outputAccumulator: seq[Output] + for outp in searchFile(searchOptC.pattern, buffer): + if outp.kind in {blockFirstMatch, blockNextMatch}: + outputAccumulator.add outp + context.add outp.pre + context.add outp.match.match + elif outp.kind == blockEnd: + outputAccumulator.add outp + context.add outp.blockEnding + # context has been formed, now check it: + if checkContext(context, searchOptC): + found = true + for output in outputAccumulator: + if optCount notin options: + yield output + else: + if output.kind in {blockFirstMatch, blockNextMatch}: + inc(cnt) + context = "" + outputAccumulator.setLen 0 + # end `if skipCheckContext`. if optCount in options and cnt > 0: yield Output(kind: justCount, matches: cnt) if yieldContents and found and optCount notin options: yield Output(kind: fileContents, buffer: move(buffer)) - -proc hasRightFileName(path: string, walkOptC: WalkOptComp[Pattern]): bool = +proc hasRightPath(path: string, walkOptC: WalkOptComp[Pattern]): bool = + if not ( + walkOpt.extensions.len > 0 or walkOpt.notExtensions.len > 0 or + walkOpt.filename.len > 0 or walkOpt.notFilename.len > 0 or + walkOpt.notDirPath.len > 0 or walkOpt.dirPath.len > 0): + return true let filename = path.lastPathPart let ex = filename.splitFile.ext.substr(1) # skip leading '.' if walkOpt.extensions.len != 0: @@ -971,31 +940,44 @@ proc hasRightFileName(path: string, walkOptC: WalkOptComp[Pattern]): bool = matched = true break if not matched: return false - for x in walkOpt.skipExtensions: + for x in walkOpt.notExtensions: if os.cmpPaths(x, ex) == 0: return false - if walkOptC.includeFile.len != 0: - var matched = false - for pat in walkOptC.includeFile: - if filename.contains(pat): - matched = true - break - if not matched: return false - for pat in walkOptC.excludeFile: - if filename.contains(pat): return false - let dirname = path.parentDir - if walkOptC.includeDir.len != 0: - var matched = false - for pat in walkOptC.includeDir: - if dirname.contains(pat): - matched = true + ensureIncluded walkOptC.filename, filename: + return false + ensureExcluded walkOptC.notFilename, filename: + return false + let parent = path.parentDir + ensureExcluded walkOptC.notDirPath, parent: + return false + ensureIncluded walkOptC.dirPath, parent: + return false + result = true + +proc isRightDirectory(path: string, walkOptC: WalkOptComp[Pattern]): bool = + ## --dirname can be only checked when the final path is known + ## so this proc is suitable for files only. + if walkOptC.dirname.len > 0: + var badDirname = false + var (nextParent, dirname) = splitPath(path) + # check that --dirname matches for one of directories in parent path: + while dirname != "": + badDirname = false + ensureIncluded walkOptC.dirname, dirname: + badDirname = true + if not badDirname: break - if not matched: return false + (nextParent, dirname) = splitPath(nextParent) + if badDirname: # badDirname was set to true for all the dirs + return false result = true -proc hasRightDirectory(path: string, walkOptC: WalkOptComp[Pattern]): bool = - let dirname = path.lastPathPart - for pat in walkOptC.excludeDir: - if dirname.contains(pat): return false +proc descendToDirectory(path: string, walkOptC: WalkOptComp[Pattern]): bool = + ## --notdirname can be checked for directories immediately for optimization to + ## prevent descending into undesired directories. + if walkOptC.notDirname.len > 0: + let dirname = path.lastPathPart + ensureExcluded walkOptC.notDirname, dirname: + return false result = true iterator walkDirBasic(dir: string, walkOptC: WalkOptComp[Pattern]): string @@ -1004,22 +986,24 @@ iterator walkDirBasic(dir: string, walkOptC: WalkOptComp[Pattern]): string var timeFiles = newSeq[(times.Time, string)]() while dirStack.len > 0: let d = dirStack.pop() + let rightDirForFiles = d.isRightDirectory(walkOptC) var files = newSeq[string]() var dirs = newSeq[string]() - for kind, path in walkDir(d): + for kind, path in walkDir(d, skipSpecial = true): case kind of pcFile: - if path.hasRightFileName(walkOptC): + if path.hasRightPath(walkOptC) and rightDirForFiles: files.add(path) of pcLinkToFile: - if optFollow in options and path.hasRightFileName(walkOptC): + if optFollow in options and path.hasRightPath(walkOptC) and + rightDirForFiles: files.add(path) of pcDir: - if optRecursive in options and path.hasRightDirectory(walkOptC): + if optRecursive in options and path.descendToDirectory(walkOptC): dirs.add path of pcLinkToDir: if optFollow in options and optRecursive in options and - path.hasRightDirectory(walkOptC): + path.descendToDirectory(walkOptC): dirs.add path if sortTime: # sort by time - collect files before yielding for file in files: @@ -1044,10 +1028,12 @@ iterator walkDirBasic(dir: string, walkOptC: WalkOptComp[Pattern]): string iterator walkRec(paths: seq[string]): tuple[error: string, filename: string] {.closure.} = declareCompiledPatterns(walkOptC, WalkOptComp): - walkOptC.excludeFile.add walkOpt.excludeFile.compileArray() - walkOptC.includeFile.add walkOpt.includeFile.compileArray() - walkOptC.includeDir.add walkOpt.includeDir.compileArray() - walkOptC.excludeDir.add walkOpt.excludeDir.compileArray() + walkOptC.notFilename.add walkOpt.notFilename.compileArray() + walkOptC.filename.add walkOpt.filename.compileArray() + walkOptC.dirname.add walkOpt.dirname.compileArray() + walkOptC.notDirname.add walkOpt.notDirname.compileArray() + walkOptC.dirPath.add walkOpt.dirPath.compileArray() + walkOptC.notDirPath.add walkOpt.notDirPath.compileArray() for path in paths: if dirExists(path): for p in walkDirBasic(path, walkOptC): @@ -1126,8 +1112,10 @@ template processFileResult(pattern: Pattern; filename: string, proc run1Thread() = declareCompiledPatterns(searchOptC, SearchOptComp): compile1Pattern(searchOpt.pattern, searchOptC.pattern) - compile1Pattern(searchOpt.checkMatch, searchOptC.checkMatch) - compile1Pattern(searchOpt.checkNoMatch, searchOptC.checkNoMatch) + searchOptC.inFile.add searchOpt.inFile.compileArray() + searchOptC.notInFile.add searchOpt.notInFile.compileArray() + searchOptC.inContext.add searchOpt.inContext.compileArray() + searchOptC.notInContext.add searchOpt.notInContext.compileArray() if optPipe in options: processFileResult(searchOptC.pattern, "-", processFile(searchOptC, "-", @@ -1169,8 +1157,10 @@ proc worker(initSearchOpt: SearchOpt) {.thread.} = searchOpt = initSearchOpt # init thread-local var declareCompiledPatterns(searchOptC, SearchOptComp): compile1Pattern(searchOpt.pattern, searchOptC.pattern) - compile1Pattern(searchOpt.checkMatch, searchOptC.checkMatch) - compile1Pattern(searchOpt.checkNoMatch, searchOptC.checkNoMatch) + searchOptC.inFile.add searchOpt.inFile.compileArray() + searchOptC.notInFile.add searchOpt.notInFile.compileArray() + searchOptC.inContext.add searchOpt.inContext.compileArray() + searchOptC.notInContext.add searchOpt.notInContext.compileArray() while true: let (fileNo, filename) = searchRequestsChan.recv() var fileResult: FileResult @@ -1268,6 +1258,11 @@ for kind, key, val in getopt(): else: paths.add(key) of cmdLongOption, cmdShortOption: + proc addNotEmpty(s: var seq[string], name: string) = + if name == "": + reportError("empty string given for option --" & key & + " (did you forget `:`?)") + s.add name case normalize(key) of "find", "f": incl(options, optFind) of "replace", "!": incl(options, optReplace) @@ -1293,15 +1288,36 @@ for kind, key, val in getopt(): nWorkers = countProcessors() else: nWorkers = parseNonNegative(val, key) - of "ext": walkOpt.extensions.add val.split('|') - of "noext", "no-ext": walkOpt.skipExtensions.add val.split('|') - of "excludedir", "exclude-dir", "ed": walkOpt.excludeDir.add val - of "includedir", "include-dir", "id": walkOpt.includeDir.add val - of "includefile", "include-file", "if": walkOpt.includeFile.add val - of "excludefile", "exclude-file", "ef": walkOpt.excludeFile.add val - of "match": searchOpt.checkMatch = val - of "nomatch": - searchOpt.checkNoMatch = val + of "extensions", "ex", "ext": walkOpt.extensions.add val.split('|') + of "nextensions", "notextensions", "nex", "notex", + "noext", "no-ext": # 2 deprecated options + walkOpt.notExtensions.add val.split('|') + of "dirname", "di": + walkOpt.dirname.addNotEmpty val + of "ndirname", "notdirname", "ndi", "notdi", + "excludedir", "exclude-dir", "ed": # 3 deprecated options + walkOpt.notDirname.addNotEmpty val + of "dirpath", "dirp", + "includedir", "include-dir", "id": # 3 deprecated options + walkOpt.dirPath.addNotEmpty val + of "ndirpath", "notdirpath", "ndirp", "notdirp": + walkOpt.notDirPath.addNotEmpty val + of "filename", "fi", + "includefile", "include-file", "if": # 3 deprecated options + walkOpt.filename.addNotEmpty val + of "nfilename", "nfi", "notfilename", "notfi", + "excludefile", "exclude-file", "ef": # 3 deprecated options + walkOpt.notFilename.addNotEmpty val + of "infile", "inf", + "matchfile", "match", "mf": # 3 deprecated options + searchOpt.inFile.addNotEmpty val + of "ninfile", "notinfile", "ninf", "notinf", + "nomatchfile", "nomatch", "nf": # 3 options are deprecated + searchOpt.notInFile.addNotEmpty val + of "incontext", "inc": + searchOpt.inContext.addNotEmpty val + of "nincontext", "notincontext", "ninc", "notinc": + searchOpt.notInContext.addNotEmpty val of "bin": case val of "on": searchOpt.checkBin = biOn diff --git a/tools/nimgrep.nim.cfg b/tools/nimgrep.nim.cfg deleted file mode 100644 index 64d3edc7a..000000000 --- a/tools/nimgrep.nim.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# don't use --gc:refc because of bug -# https://github.com/nim-lang/Nim/issues/14138 . -# --gc:orc and --gc:markandsweep work well. ---threads:on --gc:orc diff --git a/tools/niminst/buildsh.nimf b/tools/niminst/buildsh.nimf index e13221781..063a02779 100644 --- a/tools/niminst/buildsh.nimf +++ b/tools/niminst/buildsh.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateBuildShellScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated from niminst\n" & +# result = "#!/bin/sh\n# Generated from niminst\n" & # "# Template is in tools/niminst/buildsh.nimf\n" & # "# To regenerate run ``niminst csource`` or ``koch csource``\n" @@ -21,10 +21,23 @@ do optosname=$2 shift 2 ;; + --parallel) + parallel=$2 + shift 2 + ;; --extraBuildArgs) extraBuildArgs=" $2" shift 2 ;; + -h | --help) + echo "Options:" + echo " --os <OS>" + echo " --cpu <CPU architecture>" + echo " --osname <name> Additional OS specification (used for Android)" + echo " --extraBuildArgs <args> Additional arguments passed to the compiler" + echo " --parallel <number> Multiprocess build. Requires GNU parallel" + exit 0 + ;; --) # End of all options shift break; @@ -39,7 +52,15 @@ do esac done +parallel="${parallel:-0}" CC="${CC:-gcc}" +if [ "$parallel" -gt 1 ]; then + if ! command -v sem > /dev/null; then + echo "Error: GNU parallel is required to use --parallel" + exit 1 + fi + CC="sem -j $parallel --id $$ ${CC}" +fi COMP_FLAGS="${CPPFLAGS:-} ${CFLAGS:-} ?{c.ccompiler.flags}$extraBuildArgs" LINK_FLAGS="${LDFLAGS:-} ?{c.linker.flags}" PS4="" @@ -88,6 +109,11 @@ case $uos in CC="clang" LINK_FLAGS="$LINK_FLAGS -lm" ;; + *crossos* ) + myos="crossos" + CC="clang" + LINK_FLAGS="$LINK_FLAGS -lm" + ;; *openbsd* ) myos="openbsd" CC="clang" @@ -96,6 +122,7 @@ case $uos in *netbsd* ) myos="netbsd" LINK_FLAGS="$LINK_FLAGS -lm" + ucpu=`uname -p` ;; *darwin* ) myos="macosx" @@ -113,6 +140,11 @@ case $uos in myos="solaris" LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" ;; + *SunOS* ) + myos="solaris" + LINK_FLAGS="$LINK_FLAGS -ldl -lm -lsocket -lnsl" + isOpenIndiana="yes" + ;; *haiku* ) myos="haiku" LINK_FLAGS="$LINK_FLAGS -lroot -lnetwork" @@ -133,7 +165,12 @@ esac case $ucpu in *i386* | *i486* | *i586* | *i686* | *bepc* | *i86pc* ) - mycpu="i386" ;; + if [ "$isOpenIndiana" = "yes" ] || [ `uname -o` == "illumos" ] ; then + mycpu="amd64" + else + mycpu="i386" + fi + ;; *amd*64* | *x86-64* | *x86_64* ) mycpu="amd64" ;; *sparc*|*sun* ) @@ -156,8 +193,10 @@ case $ucpu in mycpu="powerpc64" ;; *power*|*ppc* ) if [ "$myos" = "freebsd" ] ; then - COMP_FLAGS="$COMP_FLAGS -m64" - LINK_FLAGS="$LINK_FLAGS -m64" + if [ "$ucpu" != "powerpc" ] ; then + COMP_FLAGS="$COMP_FLAGS -m64" + LINK_FLAGS="$LINK_FLAGS -m64" + fi mycpu=`uname -p` case $mycpu in powerpc64le) @@ -187,10 +226,14 @@ case $ucpu in mycpu="alpha" ;; *aarch64*|*arm64* ) mycpu="arm64" ;; - *arm*|*armv6l*|*armv71* ) + *arm*|*armv6l*|*armv7l*|*armv8l* ) mycpu="arm" ;; *riscv64|riscv* ) mycpu="riscv64" ;; + *e2k* ) + mycpu="e2k" ;; + *loongarch64* ) + mycpu="loongarch64" ;; *) echo 2>&1 "Error: unknown processor: $ucpu" exit 1 @@ -222,6 +265,9 @@ case $myos in $CC $COMP_FLAGS -Ic_code -c ?{f} -o ?{changeFileExt(f, "o")} # add(linkCmd, " \\\n" & changeFileExt(f, "o")) # end for + if [ "$parallel" -gt 0 ]; then + sem --wait --id $$ + fi $CC -o ?{"$binDir/" & toLowerAscii(c.name)} ?linkCmd $LINK_FLAGS ;; # end for diff --git a/tools/niminst/debcreation.nim b/tools/niminst/debcreation.nim index d0f46fa52..219cb44ce 100644 --- a/tools/niminst/debcreation.nim +++ b/tools/niminst/debcreation.nim @@ -9,6 +9,10 @@ import osproc, times, os, strutils + +when defined(nimPreviewSlimSystem): + import std/[assertions, syncio] + # http://www.debian.org/doc/manuals/maint-guide/ # Required files for debhelper. diff --git a/tools/niminst/deinstall.nimf b/tools/niminst/deinstall.nimf index 8b4477369..0dcfda75e 100644 --- a/tools/niminst/deinstall.nimf +++ b/tools/niminst/deinstall.nimf @@ -1,6 +1,6 @@ #? stdtmpl(subsChar='?') | standard #proc generateDeinstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii if [ $# -eq 1 ] ; then diff --git a/tools/niminst/install.nimf b/tools/niminst/install.nimf index a78914ecd..75ff9ce11 100644 --- a/tools/niminst/install.nimf +++ b/tools/niminst/install.nimf @@ -1,24 +1,24 @@ #? stdtmpl(subsChar = '?') | standard #proc generateInstallScript(c: ConfigData): string = -# result = "#! /bin/sh\n# Generated by niminst\n" +# result = "#!/bin/sh\n# Generated by niminst\n" # var proj = c.name.toLowerAscii set -e if [ $# -eq 1 ] ; then -# if c.cat[fcUnixBin].len > 0: - if test -f ?{c.cat[fcUnixBin][0].toUnix} +#if c.cat[fcUnixBin].len > 0: + if [ -f "?{c.cat[fcUnixBin][0].toUnix}" ] then echo "?c.displayName build detected" else echo "Please build ?c.displayName before installing it" exit 1 fi -# end if +#end if case $1 in "--help"|"-h"|"help"|"h") echo "?c.displayName installation script" - echo "Usage: [sudo] sh install.sh DIR" + echo "Usage: [sudo] [env DESTDIR=...] sh install.sh DIR" echo "Where DIR may be:" echo " /usr/bin" echo " /usr/local/bin" @@ -29,19 +29,19 @@ if [ $# -eq 1 ] ; then exit 1 ;; "/usr/bin") - bindir=/usr/bin - configdir=/etc/?proj - libdir=/usr/lib/?proj - docdir=/usr/share/?proj/doc - datadir=/usr/share/?proj/data + bindir=$1 + configdir="/etc/?proj" + libdir="/usr/lib/?proj" + docdir="/usr/share/?proj/doc" + datadir="/usr/share/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/usr/local/bin") - bindir=/usr/local/bin - configdir=/etc/?proj - libdir=/usr/local/lib/?proj - docdir=/usr/local/share/?proj/doc - datadir=/usr/local/share/?proj/data + bindir=$1 + configdir="/etc/?proj" + libdir="/usr/local/lib/?proj" + docdir="/usr/local/share/?proj/doc" + datadir="/usr/local/share/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" ;; "/opt") @@ -51,9 +51,6 @@ if [ $# -eq 1 ] ; then docdir="/opt/?proj/doc" datadir="/opt/?proj/data" nimbleDir="/opt/nimble/pkgs/?c.nimblePkgName-?c.version" - mkdir -p /opt/?proj - mkdir -p $bindir - mkdir -p $configdir ;; *) bindir="$1/?proj/bin" @@ -62,16 +59,22 @@ if [ $# -eq 1 ] ; then docdir="$1/?proj/doc" datadir="$1/?proj/data" nimbleDir="$1/?proj" - mkdir -p $1/?proj - mkdir -p $bindir - mkdir -p $configdir ;; esac - mkdir -p $libdir - mkdir -p $docdir - mkdir -p $configdir - mkdir -p $nimbleDir/ + bindir="${DESTDIR}${bindir}" + configdir="${DESTDIR}${configdir}" + libdir="${DESTDIR}${libdir}" + docdir="${DESTDIR}${docdir}" + datadir="${DESTDIR}${datadir}" + nimbleDir="${DESTDIR}${nimbleDir}" + + mkdir -p "$bindir" + mkdir -p "$configdir" + mkdir -p "$libdir" + mkdir -p "$docdir" + mkdir -p "$datadir" + mkdir -p "$nimbleDir" echo "copying files..." #var createdDirs = newStringTable() #for cat in {fcConfig..fcLib, fcNimble}: @@ -84,46 +87,46 @@ if [ $# -eq 1 ] ; then # end if # if mk.len > 0 and not createdDirs.hasKey(mk): # createdDirs[mk] = "true" - mkdir -p ?{mk.toUnix} + mkdir -p "?{mk.toUnix}" # end if # end for #end for #for f in items(c.cat[fcUnixBin]): - cp ?f.toUnix $bindir/?f.skipRoot.toUnix - chmod 755 $bindir/?f.skipRoot.toUnix + cp "?f.toUnix" "$bindir/?f.skipRoot.toUnix" + chmod 755 "$bindir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcConfig]): - cp ?f.toUnix $configdir/?f.skipRoot.toUnix - chmod 644 $configdir/?f.skipRoot.toUnix + cp "?f.toUnix" "$configdir/?f.skipRoot.toUnix" + chmod 644 "$configdir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcData]): - if [ -f ?f.toUnix ]; then - cp ?f.toUnix $datadir/?f.skipRoot.toUnix - chmod 644 $datadir/?f.skipRoot.toUnix + if [ -f "?f.toUnix" ]; then + cp "?f.toUnix" "$datadir/?f.skipRoot.toUnix" + chmod 644 "$datadir/?f.skipRoot.toUnix" fi #end for #for f in items(c.cat[fcDoc]): - if [ -f ?f.toUnix ]; then - cp ?f.toUnix $docdir/?f.skipRoot.toUnix - chmod 644 $docdir/?f.skipRoot.toUnix + if [ -f "?f.toUnix" ]; then + cp "?f.toUnix" "$docdir/?f.skipRoot.toUnix" + chmod 644 "$docdir/?f.skipRoot.toUnix" fi #end for #for f in items(c.cat[fcLib]): - cp ?f.toUnix $libdir/?f.skipRoot.toUnix - chmod 644 $libdir/?f.skipRoot.toUnix + cp "?f.toUnix" "$libdir/?f.skipRoot.toUnix" + chmod 644 "$libdir/?f.skipRoot.toUnix" #end for #for f in items(c.cat[fcNimble]): - cp ?f.toUnix $nimbleDir/?f.toUnix - chmod 644 $nimbleDir/?f.toUnix + cp "?f.toUnix" "$nimbleDir/?f.toUnix" + chmod 644 "$nimbleDir/?f.toUnix" #end for -cp ?{c.nimblePkgName}.nimble $nimbleDir/?{c.nimblePkgName}.nimble -chmod 644 $nimbleDir/?{c.nimblePkgName}.nimble +cp "?{c.nimblePkgName}.nimble" "$nimbleDir/?{c.nimblePkgName}.nimble" +chmod 644 "$nimbleDir/?{c.nimblePkgName}.nimble" echo "installation successful" else echo "?c.displayName installation script" - echo "Usage: [sudo] sh install.sh DIR" + echo "Usage: [sudo] [env DESTDIR=...] sh install.sh DIR" echo "Where DIR may be:" echo " /usr/bin" echo " /usr/local/bin" diff --git a/tools/niminst/makefile.nimf b/tools/niminst/makefile.nimf index aff0c5da0..002bc0592 100644 --- a/tools/niminst/makefile.nimf +++ b/tools/niminst/makefile.nimf @@ -16,6 +16,7 @@ endif target := ?{"$(binDir)/" & toLowerAscii(c.name)} + ucpu := $(shell sh -c 'uname -m | tr "[:upper:]" "[:lower:]"') ifeq ($(OS),Windows_NT) uos := windows @@ -25,7 +26,8 @@ endif ifeq ($(uos),linux) myos = linux - LDFLAGS += -ldl -lm + ## add -lrt to avoid "undefined reference to `clock_gettime'" with glibc<2.17 + LDFLAGS += -ldl -lm -lrt endif ifeq ($(uos),dragonfly) myos = freebsd @@ -43,6 +45,7 @@ endif ifeq ($(uos),netbsd) myos = netbsd LDFLAGS += -lm + ucpu = $(shell sh -c 'uname -p') endif ifeq ($(uos),darwin) myos = macosx @@ -102,9 +105,18 @@ endif ifeq ($(ucpu),x86_64) mycpu = amd64 endif +ifeq ($(ucpu),parisc64) + mycpu = hppa +endif +ifeq ($(ucpu),s390x) + mycpu = s390x +endif ifeq ($(ucpu),sparc) mycpu = sparc endif +ifeq ($(ucpu),sparc64) + mycpu = sparc64 +endif ifeq ($(ucpu),sun) mycpu = sparc endif @@ -130,7 +142,7 @@ ifeq ($(ucpu),powerpc) endif endif ifeq ($(ucpu),ppc) - mycpu = ppc + mycpu = powerpc endif ifneq (,$(filter $(ucpu), mips mips64)) mycpu = $(shell /bin/sh -c '"$(CC)" -dumpmachine | sed "s/-.*//"') @@ -156,14 +168,26 @@ endif ifeq ($(ucpu),armv7hl) mycpu = arm endif +ifeq ($(ucpu),armv8l) + mycpu = arm +endif ifeq ($(ucpu),aarch64) mycpu = arm64 endif +ifeq ($(ucpu),arm64) + mycpu = arm64 +endif ifeq ($(ucpu),riscv64) mycpu = riscv64 endif +ifeq ($(ucpu),e2k) + mycpu = e2k +endif +ifeq ($(ucpu),loongarch64) + mycpu = loongarch64 +endif ifndef mycpu - $(error unknown processor: $(ucpu)) + $(error unknown CPU architecture: $(ucpu) See makefile.nimf) endif # for osA in 1..c.oses.len: diff --git a/tools/niminst/nim-file.ico b/tools/niminst/nim-file.ico new file mode 100644 index 000000000..0685fedcb --- /dev/null +++ b/tools/niminst/nim-file.ico Binary files differdiff --git a/tools/niminst/niminst.nim b/tools/niminst/niminst.nim index d81b98be9..40ee79814 100644 --- a/tools/niminst/niminst.nim +++ b/tools/niminst/niminst.nim @@ -8,8 +8,15 @@ # import - os, strutils, parseopt, parsecfg, strtabs, streams, debcreation, - std / sha1 + os, strutils, parseopt, parsecfg, strtabs, streams, debcreation + +import ../../dist/checksums/src/checksums/sha1 + +when defined(nimPreviewSlimSystem): + import std/syncio + +when not defined(nimHasEffectsOf): + {.pragma: effectsOf.} const maxOS = 20 # max number of OSes @@ -161,11 +168,11 @@ proc parseCmdLine(c: var ConfigData) = next(p) var kind = p.kind var key = p.key - var val = p.val.string + var val = p.val case kind of cmdArgument: if c.actions == {}: - for a in split(normalize(key.string), {';', ','}): + for a in split(normalize(key), {';', ','}): case a of "csource": incl(c.actions, actionCSource) of "scripts": incl(c.actions, actionScripts) @@ -176,11 +183,11 @@ proc parseCmdLine(c: var ConfigData) = of "deb": incl(c.actions, actionDeb) else: quit(Usage) else: - c.infile = addFileExt(key.string, "ini") - c.nimArgs = cmdLineRest(p).string + c.infile = addFileExt(key, "ini") + c.nimArgs = cmdLineRest(p) break of cmdLongOption, cmdShortOption: - case normalize(key.string) + case normalize(key) of "help", "h": stdout.write(Usage) quit(0) @@ -198,7 +205,7 @@ proc parseCmdLine(c: var ConfigData) = if c.infile.len == 0: quit(Usage) if c.mainfile.len == 0: c.mainfile = changeFileExt(c.infile, "nim") -proc eqT(a, b: string; t: proc (a: char): char{.nimcall.}): bool = +proc eqT(a, b: string; t: proc (a: char): char {.nimcall.}): bool {.effectsOf: t.} = ## equality under a transformation ``t``. candidate for the stdlib? var i = 0 var j = 0 @@ -508,6 +515,17 @@ template gatherFiles(fun, libpath, outDir) = # commenting out for now, see discussion in https://github.com/nim-lang/Nim/pull/13413 # copySrc(libpath / "lib/wrappers/linenoise/linenoise.h") +proc exe(f: string): string = + result = addFileExt(f, ExeExt) + when defined(windows): + result = result.replace('/','\\') + +proc findNim(): string = + let nim = "nim".exe + result = quoteShell("bin" / nim) + if not fileExists(result): + result = "nim" + proc srcdist(c: var ConfigData) = let cCodeDir = getOutputDir(c) / "c_code" if not dirExists(cCodeDir): createDir(cCodeDir) @@ -526,10 +544,10 @@ proc srcdist(c: var ConfigData) = var dir = getOutputDir(c) / buildDir(osA, cpuA) if dirExists(dir): removeDir(dir) createDir(dir) - var cmd = ("nim compile -f --symbolfiles:off --compileonly " & + var cmd = ("$# compile -f --incremental:off --compileonly " & "--gen_mapping --cc:gcc --skipUserCfg" & " --os:$# --cpu:$# $# $#") % - [osname, cpuname, c.nimArgs, c.mainfile] + [findNim(), osname, cpuname, c.nimArgs, c.mainfile] echo(cmd) if execShellCmd(cmd) != 0: quit("Error: call to nim compiler failed") diff --git a/tools/niminst/setup.ico b/tools/niminst/setup.ico new file mode 100644 index 000000000..867163046 --- /dev/null +++ b/tools/niminst/setup.ico Binary files differdiff --git a/tools/niminst/uninstall.ico b/tools/niminst/uninstall.ico new file mode 100644 index 000000000..aff054644 --- /dev/null +++ b/tools/niminst/uninstall.ico Binary files differdiff --git a/tools/nimweb.nim b/tools/nimweb.nim deleted file mode 100644 index f71b5f3be..000000000 --- a/tools/nimweb.nim +++ /dev/null @@ -1,556 +0,0 @@ -# -# -# Nim Website Generator -# (c) Copyright 2015 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import - os, strutils, times, parseopt, parsecfg, streams, strtabs, tables, - re, htmlgen, macros, md5, osproc, parsecsv, algorithm - -from xmltree import escape - -type - TKeyValPair = tuple[key, id, val: string] - TConfigData = object of RootObj - tabs, links: seq[TKeyValPair] - doc, srcdoc, srcdoc2, webdoc, pdf: seq[string] - authors, projectName, projectTitle, logo, infile, ticker: string - vars: StringTableRef - nimCompiler: string - nimArgs: string - gitURL: string - docHTMLOutput: string - webUploadOutput: string - quotations: Table[string, tuple[quote, author: string]] - numProcessors: int # Set by parallelBuild:n, only works for values > 0. - gaId: string # google analytics ID, nil means analytics are disabled - TRssItem = object - year, month, day, title, url, content: string - TAction = enum - actAll, actOnlyWebsite, actPdf, actJson2, actOnlyDocs - - Sponsor = object - logo: string - name: string - url: string - thisMonth: int - allTime: int - since: string - level: int - -var action: TAction - -proc initConfigData(c: var TConfigData) = - c.tabs = @[] - c.links = @[] - c.doc = @[] - c.srcdoc = @[] - c.srcdoc2 = @[] - c.webdoc = @[] - c.pdf = @[] - c.infile = "" - c.nimArgs = "--hint[Conf]:off --hint[Path]:off --hint[Processing]:off -d:boot " - c.gitURL = "https://github.com/nim-lang/Nim" - c.docHTMLOutput = "doc/html" - c.webUploadOutput = "web/upload" - c.authors = "" - c.projectTitle = "" - c.projectName = "" - c.logo = "" - c.ticker = "" - c.vars = newStringTable(modeStyleInsensitive) - c.numProcessors = countProcessors() - # Attempts to obtain the git current commit. - when false: - let (output, code) = execCmdEx("git log -n 1 --format=%H") - if code == 0 and output.strip.len == 40: - c.gitCommit = output.strip - c.quotations = initTable[string, tuple[quote, author: string]]() - -include "website.nimf" - -# ------------------------- configuration file ------------------------------- - -const - version = "0.8" - usage = "nimweb - Nim Website Generator Version " & version & """ - - (c) 2015 Andreas Rumpf -Usage: - nimweb [options] ini-file[.ini] [compile_options] -Options: - -h, --help shows this help - -v, --version shows the version - -o, --output overrides output directory instead of default - web/upload and doc/html - --nimCompiler overrides nim compiler; default = bin/nim - --var:name=value set the value of a variable - --website only build the website, not the full documentation - --pdf build the PDF version of the documentation - --json2 build JSON of the documentation - --onlyDocs build only the documentation - --git.url override base url in generated doc links - --git.commit override commit/branch in generated doc links 'source' - --git.devel override devel branch in generated doc links 'edit' -Compile_options: - will be passed to the Nim compiler -""" - - rYearMonthDay = r"on\s+(\d{2})\/(\d{2})\/(\d{4})" - rssUrl = "http://nim-lang.org/news.xml" - rssNewsUrl = "http://nim-lang.org/news.html" - activeSponsors = "web/sponsors.csv" - inactiveSponsors = "web/inactive_sponsors.csv" - validAnchorCharacters = Letters + Digits - - -macro id(e: untyped): untyped = - ## generates the rss xml ``id`` element. - let e = callsite() - result = xmlCheckedTag(e, "id") - -macro updated(e: varargs[untyped]): untyped = - ## generates the rss xml ``updated`` element. - let e = callsite() - result = xmlCheckedTag(e, "updated") - -proc updatedDate(year, month, day: string): string = - ## wrapper around the update macro with easy input. - result = updated("$1-$2-$3T00:00:00Z" % [year, - repeat("0", 2 - len(month)) & month, - repeat("0", 2 - len(day)) & day]) - -macro entry(e: varargs[untyped]): untyped = - ## generates the rss xml ``entry`` element. - let e = callsite() - result = xmlCheckedTag(e, "entry") - -macro content(e: varargs[untyped]): untyped = - ## generates the rss xml ``content`` element. - let e = callsite() - result = xmlCheckedTag(e, "content", reqAttr = "type") - -proc parseCmdLine(c: var TConfigData) = - var p = initOptParser() - while true: - next(p) - var kind = p.kind - var key = p.key - var val = p.val - case kind - of cmdArgument: - c.infile = addFileExt(key, "ini") - c.nimArgs.add(cmdLineRest(p)) - break - of cmdLongOption, cmdShortOption: - case normalize(key) - of "help", "h": - stdout.write(usage) - quit(0) - of "version", "v": - stdout.write(version & "\n") - quit(0) - of "output", "o": - c.webUploadOutput = val - c.docHTMLOutput = val / "docs" - of "nimcompiler": - c.nimCompiler = val - of "parallelbuild": - try: - let num = parseInt(val) - if num != 0: c.numProcessors = num - except ValueError: - quit("invalid numeric value for --parallelBuild") - of "var": - var idx = val.find('=') - if idx < 0: quit("invalid command line") - c.vars[substr(val, 0, idx-1)] = substr(val, idx+1) - of "website": action = actOnlyWebsite - of "pdf": action = actPdf - of "json2": action = actJson2 - of "onlydocs": action = actOnlyDocs - of "googleanalytics": - c.gaId = val - c.nimArgs.add("--doc.googleAnalytics:" & val & " ") - of "git.url": - c.gitURL = val - of "git.commit": - c.nimArgs.add("--git.commit:" & val & " ") - of "git.devel": - c.nimArgs.add("--git.devel:" & val & " ") - else: - echo("Invalid argument '$1'" % [key]) - quit(usage) - of cmdEnd: break - if c.infile.len == 0: quit(usage) - -proc walkDirRecursively(s: var seq[string], root, ext: string) = - for k, f in walkDir(root): - case k - of pcFile, pcLinkToFile: - if cmpIgnoreCase(ext, splitFile(f).ext) == 0: - add(s, f) - of pcDir: walkDirRecursively(s, f, ext) - of pcLinkToDir: discard - -proc addFiles(s: var seq[string], dir, ext: string, patterns: seq[string]) = - for p in items(patterns): - if fileExists(dir / addFileExt(p, ext)): - s.add(dir / addFileExt(p, ext)) - if dirExists(dir / p): - walkDirRecursively(s, dir / p, ext) - -proc parseIniFile(c: var TConfigData) = - var - p: CfgParser - section: string # current section - var input = newFileStream(c.infile, fmRead) - if input == nil: quit("cannot open: " & c.infile) - open(p, input, c.infile) - while true: - var k = next(p) - case k.kind - of cfgEof: break - of cfgSectionStart: - section = normalize(k.section) - case section - of "project", "links", "tabs", "ticker", "documentation", "var": discard - else: echo("[Warning] Skipping unknown section: " & section) - - of cfgKeyValuePair: - var v = k.value % c.vars - c.vars[k.key] = v - - case section - of "project": - case normalize(k.key) - of "name": c.projectName = v - of "title": c.projectTitle = v - of "logo": c.logo = v - of "authors": c.authors = v - else: quit(errorStr(p, "unknown variable: " & k.key)) - of "var": discard - of "links": - let valID = v.split(';') - add(c.links, (k.key.replace('_', ' '), valID[1], valID[0])) - of "tabs": add(c.tabs, (k.key, "", v)) - of "ticker": c.ticker = v - of "documentation": - case normalize(k.key) - of "doc": addFiles(c.doc, "doc", ".rst", split(v, {';'})) - of "pdf": addFiles(c.pdf, "doc", ".rst", split(v, {';'})) - of "srcdoc": addFiles(c.srcdoc, "lib", ".nim", split(v, {';'})) - of "srcdoc2": addFiles(c.srcdoc2, "lib", ".nim", split(v, {';'})) - of "webdoc": addFiles(c.webdoc, "lib", ".nim", split(v, {';'})) - of "parallelbuild": - try: - let num = parseInt(v) - if num != 0: c.numProcessors = num - except ValueError: - quit("invalid numeric value for --parallelBuild in config") - else: quit(errorStr(p, "unknown variable: " & k.key)) - of "quotations": - let vSplit = v.split('-') - doAssert vSplit.len == 2 - c.quotations[k.key.normalize] = (vSplit[0], vSplit[1]) - else: discard - of cfgOption: quit(errorStr(p, "syntax error")) - of cfgError: quit(errorStr(p, k.msg)) - close(p) - if c.projectName.len == 0: - c.projectName = changeFileExt(extractFilename(c.infile), "") - -# ------------------- main ---------------------------------------------------- - - -proc exe(f: string): string = return addFileExt(f, ExeExt) - -proc findNim(c: TConfigData): string = - if c.nimCompiler.len > 0: return c.nimCompiler - var nim = "nim".exe - result = "bin" / nim - if fileExists(result): return - for dir in split(getEnv("PATH"), PathSep): - if fileExists(dir / nim): return dir / nim - # assume there is a symlink to the exe or something: - return nim - -proc exec(cmd: string) = - echo(cmd) - let (outp, exitCode) = osproc.execCmdEx(cmd) - if exitCode != 0: quit outp - -proc sexec(cmds: openarray[string]) = - ## Serial queue wrapper around exec. - for cmd in cmds: exec(cmd) - -proc mexec(cmds: openarray[string], processors: int) = - ## Multiprocessor version of exec - doAssert processors > 0, "nimweb needs at least one processor" - if processors == 1: - sexec(cmds) - return - let r = execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}, - n = processors) - if r != 0: - echo "external program failed, retrying serial work queue for logs!" - sexec(cmds) - -proc buildDocSamples(c: var TConfigData, destPath: string) = - ## Special case documentation sample proc. - ## - ## TODO: consider integrating into the existing generic documentation builders - ## now that we have a single `doc` command. - exec(findNim(c) & " doc $# -o:$# $#" % - [c.nimArgs, destPath / "docgen_sample.html", "doc" / "docgen_sample.nim"]) - -proc pathPart(d: string): string = splitFile(d).dir.replace('\\', '/') - -proc buildDoc(c: var TConfigData, destPath: string) = - # call nim for the documentation: - var - commands = newSeq[string](len(c.doc) + len(c.srcdoc) + len(c.srcdoc2)) - i = 0 - for d in items(c.doc): - commands[i] = findNim(c) & " rst2html $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(c.srcdoc): - commands[i] = findNim(c) & " doc0 $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - for d in items(c.srcdoc2): - commands[i] = findNim(c) & " doc $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(d).name, "html"), d] - i.inc - - mexec(commands, c.numProcessors) - exec(findNim(c) & " buildIndex -o:$1/theindex.html $1" % [destPath]) - -proc buildPdfDoc(c: var TConfigData, destPath: string) = - createDir(destPath) - if os.execShellCmd("pdflatex -version") != 0: - echo "pdflatex not found; no PDF documentation generated" - else: - const pdflatexcmd = "pdflatex -interaction=nonstopmode " - for d in items(c.pdf): - exec(findNim(c) & " rst2tex $# $#" % [c.nimArgs, d]) - # call LaTeX twice to get cross references right: - exec(pdflatexcmd & changeFileExt(d, "tex")) - exec(pdflatexcmd & changeFileExt(d, "tex")) - # delete all the crappy temporary files: - let pdf = splitFile(d).name & ".pdf" - let dest = destPath / pdf - removeFile(dest) - moveFile(dest=dest, source=pdf) - removeFile(changeFileExt(pdf, "aux")) - if fileExists(changeFileExt(pdf, "toc")): - removeFile(changeFileExt(pdf, "toc")) - removeFile(changeFileExt(pdf, "log")) - removeFile(changeFileExt(pdf, "out")) - removeFile(changeFileExt(d, "tex")) - -proc buildAddDoc(c: var TConfigData, destPath: string) = - # build additional documentation (without the index): - var commands = newSeq[string](c.webdoc.len) - for i, doc in pairs(c.webdoc): - commands[i] = findNim(c) & " doc $# --git.url:$# -o:$# $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(splitFile(doc).name, "html"), doc] - mexec(commands, c.numProcessors) - -proc parseNewsTitles(inputFilename: string): seq[TRssItem] = - # Goes through each news file, returns its date/title. - result = @[] - var matches: array[3, string] - let reYearMonthDay = re(rYearMonthDay) - for kind, path in walkDir(inputFilename): - let (dir, name, ext) = path.splitFile - if ext == ".rst": - let content = readFile(path) - let title = content.splitLines()[0] - let urlPath = "news/" & name & ".html" - if content.find(reYearMonthDay, matches) >= 0: - result.add(TRssItem(year: matches[2], month: matches[1], day: matches[0], - title: title, url: "http://nim-lang.org/" & urlPath, - content: content)) - result.reverse() - -proc genUUID(text: string): string = - # Returns a valid RSS uuid, which is basically md5 with dashes and a prefix. - result = getMD5(text) - result.insert("-", 20) - result.insert("-", 16) - result.insert("-", 12) - result.insert("-", 8) - result.insert("urn:uuid:") - -proc genNewsLink(title: string): string = - # Mangles a title string into an expected news.html anchor. - result = title - result.insert("Z") - for i in 1..len(result)-1: - let letter = result[i].toLowerAscii() - if letter in validAnchorCharacters: - result[i] = letter - else: - result[i] = '-' - result.insert(rssNewsUrl & "#") - -proc generateRss(outputFilename: string, news: seq[TRssItem]) = - # Given a list of rss items generates an rss overwriting destination. - var - output: File - - if not open(output, outputFilename, mode = fmWrite): - quit("Could not write to $1 for rss generation" % [outputFilename]) - defer: output.close() - - output.write("""<?xml version="1.0" encoding="utf-8"?> -<feed xmlns="http://www.w3.org/2005/Atom"> -""") - output.write(title("Nim website news")) - output.write(link(href = rssUrl, rel = "self")) - output.write(link(href = rssNewsUrl)) - output.write(id(rssNewsUrl)) - - let now = utc(getTime()) - output.write(updatedDate($now.year, $(int(now.month) + 1), $now.monthday)) - - for rss in news: - output.write(entry( - title(xmltree.escape(rss.title)), - id(genUUID(rss.title)), - link(`type` = "text/html", rel = "alternate", - href = rss.url), - updatedDate(rss.year, rss.month, rss.day), - "<author><name>Nim</name></author>", - content(xmltree.escape(rss.content), `type` = "text") - )) - - output.write("""</feed>""") - -proc buildNewsRss(c: var TConfigData, destPath: string) = - # generates an xml feed from the web/news.rst file - let - srcFilename = "web" / "news" - destFilename = destPath / changeFileExt(splitFile(srcFilename).name, "xml") - - generateRss(destFilename, parseNewsTitles(srcFilename)) - -proc readSponsors(sponsorsFile: string): seq[Sponsor] = - result = @[] - var fileStream = newFileStream(sponsorsFile, fmRead) - if fileStream == nil: quit("Cannot open sponsors.csv file: " & sponsorsFile) - var parser: CsvParser - open(parser, fileStream, sponsorsFile) - discard readRow(parser) # Skip the header row. - while readRow(parser): - result.add(Sponsor(logo: parser.row[0], name: parser.row[1], - url: parser.row[2], thisMonth: parser.row[3].parseInt, - allTime: parser.row[4].parseInt, - since: parser.row[5], level: parser.row[6].parseInt)) - parser.close() - -proc buildSponsors(c: var TConfigData, outputDir: string) = - let sponsors = generateSponsorsPage(readSponsors(activeSponsors), - readSponsors(inactiveSponsors)) - let outFile = outputDir / "sponsors.html" - var f: File - if open(f, outFile, fmWrite): - writeLine(f, generateHtmlPage(c, "", "Our Sponsors", sponsors, "")) - close(f) - else: - quit("[Error] Cannot write file: " & outFile) - -const - cmdRst2Html = " rst2html --compileonly $1 -o:web/$2.temp web/$2.rst" - -proc buildPage(c: var TConfigData, file, title, rss: string, assetDir = "") = - exec(findNim(c) & cmdRst2Html % [c.nimArgs, file]) - var temp = "web" / changeFileExt(file, "temp") - var content: string - try: - content = readFile(temp) - except IOError: - quit("[Error] cannot open: " & temp) - var f: File - var outfile = c.webUploadOutput / "$#.html" % file - if not dirExists(outfile.splitFile.dir): - createDir(outfile.splitFile.dir) - if open(f, outfile, fmWrite): - writeLine(f, generateHTMLPage(c, file, title, content, rss, assetDir)) - close(f) - else: - quit("[Error] cannot write file: " & outfile) - removeFile(temp) - -proc buildNews(c: var TConfigData, newsDir: string, outputDir: string) = - for kind, path in walkDir(newsDir): - let (dir, name, ext) = path.splitFile - if ext == ".rst": - let title = readFile(path).splitLines()[0] - buildPage(c, tailDir(dir) / name, title, "", "../") - else: - echo("Skipping file in news directory: ", path) - -proc buildWebsite(c: var TConfigData) = - if c.ticker.len > 0: - try: - c.ticker = readFile("web" / c.ticker) - except IOError: - quit("[Error] cannot open: " & c.ticker) - for i in 0..c.tabs.len-1: - var file = c.tabs[i].val - let rss = if file in ["news", "index"]: extractFilename(rssUrl) else: "" - if '.' in file: continue - buildPage(c, file, if file == "question": "FAQ" else: file, rss) - copyDir("web/assets", c.webUploadOutput / "assets") - buildNewsRss(c, c.webUploadOutput) - buildSponsors(c, c.webUploadOutput) - buildNews(c, "web/news", c.webUploadOutput / "news") - -proc onlyDocs(c: var TConfigData) = - createDir(c.docHTMLOutput) - buildDocSamples(c, c.docHTMLOutput) - buildDoc(c, c.docHTMLOutput) - -proc main(c: var TConfigData) = - buildWebsite(c) - let docup = c.webUploadOutput / NimVersion - createDir(docup) - buildAddDoc(c, docup) - buildDocSamples(c, docup) - buildDoc(c, docup) - onlyDocs(c) - -proc json2(c: var TConfigData) = - const destPath = "web/json2" - var commands = newSeq[string](c.srcdoc2.len) - var i = 0 - for d in items(c.srcdoc2): - createDir(destPath / splitFile(d).dir) - commands[i] = findNim(c) & " jsondoc $# --git.url:$# -o:$# --index:on $#" % - [c.nimArgs, c.gitURL, - destPath / changeFileExt(d, "json"), d] - i.inc - - mexec(commands, c.numProcessors) - -var c: TConfigData -initConfigData(c) -parseCmdLine(c) -parseIniFile(c) -case action -of actOnlyWebsite: buildWebsite(c) -of actPdf: buildPdfDoc(c, "doc/pdf") -of actOnlyDocs: onlyDocs(c) -of actAll: main(c) -of actJson2: json2(c) diff --git a/tools/officialpackages.nim b/tools/officialpackages.nim new file mode 100644 index 000000000..633944a14 --- /dev/null +++ b/tools/officialpackages.nim @@ -0,0 +1,21 @@ +import std/[strformat, paths, dirs, envvars] +from std/os import execShellCmd + +proc exec*(cmd: string, errorcode: int = QuitFailure, additionalPath = "") = + let prevPath = getEnv("PATH") + if additionalPath.len > 0: + var absolute = Path(additionalPath) + if not absolute.isAbsolute: + absolute = getCurrentDir() / absolute + echo("Adding to $PATH: ", string(absolute)) + putEnv("PATH", (if prevPath.len > 0: prevPath & PathSep else: "") & string(absolute)) + echo(cmd) + if execShellCmd(cmd) != 0: quit("FAILURE", errorcode) + putEnv("PATH", prevPath) + +proc gitClonePackages*(names: seq[string]) = + if not dirExists(Path"pkgs"): + createDir(Path"pkgs") + for name in names: + if not dirExists(Path"pkgs" / Path(name)): + exec fmt"git clone https://github.com/nim-lang/{name} pkgs/{name}" diff --git a/tools/trimcc.nim b/tools/trimcc.nim index 959eaf310..41795e1aa 100644 --- a/tools/trimcc.nim +++ b/tools/trimcc.nim @@ -31,7 +31,7 @@ proc processIncludes(dir: string, whitelist: StringTableRef) = if ('.' notin name and "include" in path) or ("c++" in path): let n = whitelist.getOrDefault(name) if n != "processed": whitelist[name] = "found" - if name.endswith(".h"): + if name.endsWith(".h"): let n = whitelist.getOrDefault(name) if n == "found": includes(path, name, whitelist) of pcDir: processIncludes(path, whitelist) diff --git a/tools/unicode_parsedata.nim b/tools/unicode_parsedata.nim index cca377f51..bd12998d1 100644 --- a/tools/unicode_parsedata.nim +++ b/tools/unicode_parsedata.nim @@ -26,34 +26,54 @@ var proc parseData(data: seq[string]) = - for line in data: - let - fields = line.split(';') - code = fields[0].parseHexInt() - category = fields[2] - uc = fields[12] - lc = fields[13] - tc = fields[14] - + proc doAdd(firstCode, lastCode: int, category, uc, lc, tc: string) = if category notin spaces and category notin letters: - continue + return + if firstCode != lastCode: + doAssert uc == "" and lc == "" and tc == "" if uc.len > 0: - let diff = 500 + uc.parseHexInt() - code - toUpper.add (code, diff) + let diff = 500 + uc.parseHexInt() - firstCode + toUpper.add (firstCode, diff) if lc.len > 0: - let diff = 500 + lc.parseHexInt() - code - toLower.add (code, diff) + let diff = 500 + lc.parseHexInt() - firstCode + toLower.add (firstCode, diff) if tc.len > 0 and tc != uc: # if titlecase is different than uppercase - let diff = 500 + tc.parseHexInt() - code + let diff = 500 + tc.parseHexInt() - firstCode if diff != 500: - toTitle.add (code, diff) + toTitle.add (firstCode, diff) - if category in spaces: - unispaces.add code + for code in firstCode..lastCode: + if category in spaces: + unispaces.add code + else: + alphas.add code + + var idx = 0 + while idx < data.len: + let + line = data[idx] + fields = line.split(';') + code = fields[0].parseHexInt() + name = fields[1] + category = fields[2] + uc = fields[12] + lc = fields[13] + tc = fields[14] + inc(idx) + if name.endsWith(", First>"): + doAssert idx < data.len + let + nextLine = data[idx] + nextFields = nextLine.split(';') + nextCode = nextFields[0].parseHexInt() + nextName = nextFields[1] + inc(idx) + doAssert nextName.endsWith(", Last>") + doAdd(code, nextCode, category, uc, lc, tc) else: - alphas.add code + doAdd(code, code, category, uc, lc, tc) proc splitRanges(a: seq[Singlets], r: var seq[Ranges], s: var seq[Singlets]) = ## Splits `toLower`, `toUpper` and `toTitle` into separate sequences: @@ -153,18 +173,18 @@ proc createHeader(output: var string) = proc `$`(r: Ranges): string = let - start = "0x" & toHex(r.start, 5) - stop = "0x" & toHex(r.stop, 5) + start = "0x" & toHex(r.start, 5) & "'i32" + stop = "0x" & toHex(r.stop, 5) & "'i32" result = "$#, $#, $#,\n" % [start, stop, $r.diff] proc `$`(r: Singlets): string = - let code = "0x" & toHex(r.code, 5) + let code = "0x" & toHex(r.code, 5) & "'i32" result = "$#, $#,\n" % [code, $r.diff] proc `$`(r: NonLetterRanges): string = let - start = "0x" & toHex(r.start, 5) - stop = "0x" & toHex(r.stop, 5) + start = "0x" & toHex(r.start, 5) & "'i32" + stop = "0x" & toHex(r.stop, 5) & "'i32" result = "$#, $#,\n" % [start, stop] @@ -178,7 +198,7 @@ proc outputSeq(s: seq[Ranges|Singlets|NonLetterRanges], name: string, proc outputSeq(s: seq[int], name: string, output: var string) = output.add " $# = [\n" % name for i in s: - output.add " 0x$#,\n" % toHex(i, 5) + output.add " 0x$#'i32,\n" % toHex(i, 5) output.add " ]\n\n" proc outputSpaces(s: seq[int], name: string, output: var string) = diff --git a/tools/urldownloader.nim b/tools/urldownloader.nim deleted file mode 100644 index 73e4034c9..000000000 --- a/tools/urldownloader.nim +++ /dev/null @@ -1,431 +0,0 @@ - -# -# -# Windows native FTP/HTTP/HTTPS file downloader -# (c) Copyright 2017 Eugene Kabanov -# -# See the file "LICENSE", included in this -# distribution, for details about the copyright. -# - -## This module implements native Windows FTP/HTTP/HTTPS downloading feature, -## using ``urlmon.UrlDownloadToFile()``. -## -## - -when not (defined(windows) or defined(nimdoc)): - {.error: "Platform is not supported.".} - -import os - -type - DownloadOptions* = enum - ## Available download options - optUseCache, ## Use Windows cache. - optUseProgressCallback, ## Report progress via callback. - optIgnoreSecurity ## Ignore HTTPS security problems. - - DownloadStatus* = enum - ## Available download status sent to ``progress`` callback. - statusProxyDetecting, ## Automatic Proxy detection. - statusCookieSent ## Cookie will be sent with request. - statusResolving, ## Resolving URL with DNS. - statusConnecting, ## Establish connection to server. - statusRedirecting ## HTTP redirection pending. - statusRequesting, ## Sending request to server. - statusMimetypeAvailable, ## Mimetype received from server. - statusBeginDownloading, ## Download process starting. - statusDownloading, ## Download process pending. - statusEndDownloading, ## Download process finished. - statusCacheAvailable ## File found in Windows cache. - statusUnsupported ## Unsupported status. - statusError ## Error happens. - - DownloadProgressCallback* = proc(status: DownloadStatus, progress: uint, - progressMax: uint, - message: string) - ## Progress callback. - ## - ## status - ## Indicate current stage of downloading process. - ## - ## progress - ## Number of bytes currently downloaded. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## progressMax - ## Number of bytes expected to download. Available only, if ``status`` is - ## ``statusBeginDownloading``, ``statusDownloading`` or - ## ``statusEndDownloading``. - ## - ## message - ## Status message, which depends on ``status`` code. - ## - ## Available messages' values: - ## - ## statusResolving - ## URL hostname to be resolved. - ## statusConnecting - ## IP address - ## statusMimetypeAvailable - ## Downloading resource MIME type. - ## statusCacheAvailable - ## Path to filename stored in Windows cache. - -type - UUID = array[4, uint32] - - LONG = clong - ULONG = culong - HRESULT = clong - DWORD = uint32 - OLECHAR = uint16 - OLESTR = ptr OLECHAR - LPWSTR = OLESTR - UINT = cuint - REFIID = ptr UUID - -const - E_NOINTERFACE = 0x80004002'i32 - E_NOTIMPL = 0x80004001'i32 - S_OK = 0x00000000'i32 - - CP_UTF8 = 65001'u32 - - IID_IUnknown = UUID([0'u32, 0'u32, 192'u32, 1174405120'u32]) - IID_IBindStatusCallback = UUID([2045430209'u32, 298760953'u32, - 2852160140'u32, 195644160'u32]) - - BINDF_GETNEWESTVERSION = 0x00000010'u32 - BINDF_IGNORESECURITYPROBLEM = 0x00000100'u32 - BINDF_RESYNCHRONIZE = 0x00000200'u32 - BINDF_NO_UI = 0x00000800'u32 - BINDF_SILENTOPERATION = 0x00001000'u32 - BINDF_PRAGMA_NO_CACHE = 0x00002000'u32 - - ERROR_FILE_NOT_FOUND = 2 - ERROR_ACCESS_DENIED = 5 - - BINDSTATUS_FINDINGRESOURCE = 1 - BINDSTATUS_CONNECTING = 2 - BINDSTATUS_REDIRECTING = 3 - BINDSTATUS_BEGINDOWNLOADDATA = 4 - BINDSTATUS_DOWNLOADINGDATA = 5 - BINDSTATUS_ENDDOWNLOADDATA = 6 - BINDSTATUS_SENDINGREQUEST = 11 - BINDSTATUS_MIMETYPEAVAILABLE = 13 - BINDSTATUS_CACHEFILENAMEAVAILABLE = 14 - BINDSTATUS_PROXYDETECTING = 32 - BINDSTATUS_COOKIE_SENT = 34 - -type - STGMEDIUM = object - tymed: DWORD - pstg: pointer - pUnkForRelease: pointer - - SECURITY_ATTRIBUTES = object - nLength*: uint32 - lpSecurityDescriptor*: pointer - bInheritHandle*: int32 - - BINDINFO = object - cbSize: ULONG - stgmedData: STGMEDIUM - szExtraInfo: LPWSTR - grfBindInfoF: DWORD - dwBindVerb: DWORD - szCustomVerb: LPWSTR - cbstgmedData: DWORD - dwOptions: DWORD - dwOptionsFlags: DWORD - dwCodePage: DWORD - securityAttributes: SECURITY_ATTRIBUTES - iid: UUID - pUnk: pointer - dwReserved: DWORD - - IBindStatusCallback = object - vtable: ptr IBindStatusCallbackVTable - options: set[DownloadOptions] - objectRefCount: ULONG - binfoFlags: DWORD - progressCallback: DownloadProgressCallback - - PIBindStatusCallback = ptr IBindStatusCallback - LPBINDSTATUSCALLBACK = PIBindStatusCallback - - IBindStatusCallbackVTable = object - QueryInterface: proc (self: PIBindStatusCallback, - riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} - AddRef: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - Release: proc(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} - OnStartBinding: proc(self: PIBindStatusCallback, - dwReserved: DWORD, pib: pointer): HRESULT - {.gcsafe, stdcall.} - GetPriority: proc(self: PIBindStatusCallback, pnPriority: ptr LONG): HRESULT - {.gcsafe, stdcall.} - OnLowResource: proc(self: PIBindStatusCallback, dwReserved: DWORD): HRESULT - {.gcsafe, stdcall.} - OnProgress: proc(self: PIBindStatusCallback, ulProgress: ULONG, - ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT - {.gcsafe, stdcall.} - OnStopBinding: proc(self: PIBindStatusCallback, hresult: HRESULT, - szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} - GetBindInfo: proc(self: PIBindStatusCallback, grfBINDF: ptr DWORD, - pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} - OnDataAvailable: proc(self: PIBindStatusCallback, grfBSCF: DWORD, - dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT - {.gcsafe, stdcall.} - OnObjectAvailable: proc(self: PIBindStatusCallback, riid: REFIID, - punk: pointer): HRESULT - {.gcsafe, stdcall.} - -template FAILED(hr: HRESULT): bool = - (hr < 0) - -proc URLDownloadToFile(pCaller: pointer, szUrl: LPWSTR, szFileName: LPWSTR, - dwReserved: DWORD, - lpfnCb: LPBINDSTATUSCALLBACK): HRESULT - {.stdcall, dynlib: "urlmon.dll", importc: "URLDownloadToFileW".} - -proc WideCharToMultiByte(CodePage: UINT, dwFlags: DWORD, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpDefaultChar: ptr char, - lpUsedDefaultChar: ptr uint32): cint - {.stdcall, dynlib: "kernel32.dll", importc: "WideCharToMultiByte".} - -proc MultiByteToWideChar(CodePage: UINT, dwFlags: DWORD, - lpMultiByteStr: ptr char, cbMultiByte: cint, - lpWideCharStr: ptr OLECHAR, cchWideChar: cint): cint - {.stdcall, dynlib: "kernel32.dll", importc: "MultiByteToWideChar".} -proc DeleteUrlCacheEntry(lpszUrlName: LPWSTR): int32 - {.stdcall, dynlib: "wininet.dll", importc: "DeleteUrlCacheEntryW".} - -proc `==`(a, b: UUID): bool = - result = false - if a[0] == b[0] and a[1] == b[1] and - a[2] == b[2] and a[3] == b[3]: - result = true - -proc `$`(bstr: LPWSTR): string = - var buffer: char - var count = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(buffer), 0, - nil, nil) - if count == 0: - raiseOsError(osLastError()) - else: - result = newString(count + 8) - let res = WideCharToMultiByte(CP_UTF8, 0, bstr, -1, addr(result[0]), count, - nil, nil) - if res == 0: - raiseOsError(osLastError()) - result.setLen(res - 1) - -proc toBstring(str: string): LPWSTR = - var buffer: OLECHAR - var count = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - addr(buffer), 0) - if count == 0: - raiseOsError(osLastError()) - else: - result = cast[LPWSTR](alloc0((count + 1) * sizeof(OLECHAR))) - let res = MultiByteToWideChar(CP_UTF8, 0, unsafeAddr(str[0]), -1, - result, count) - if res == 0: - raiseOsError(osLastError()) - -proc freeBstring(bstr: LPWSTR) = - dealloc(bstr) - -proc getStatus(scode: ULONG): DownloadStatus = - case scode - of 0: result = statusError - of BINDSTATUS_PROXYDETECTING: result = statusProxyDetecting - of BINDSTATUS_REDIRECTING: result = statusRedirecting - of BINDSTATUS_COOKIE_SENT: result = statusCookieSent - of BINDSTATUS_FINDINGRESOURCE: result = statusResolving - of BINDSTATUS_CONNECTING: result = statusConnecting - of BINDSTATUS_SENDINGREQUEST: result = statusRequesting - of BINDSTATUS_MIMETYPEAVAILABLE: result = statusMimetypeAvailable - of BINDSTATUS_BEGINDOWNLOADDATA: result = statusBeginDownloading - of BINDSTATUS_DOWNLOADINGDATA: result = statusDownloading - of BINDSTATUS_ENDDOWNLOADDATA: result = statusEndDownloading - of BINDSTATUS_CACHEFILENAMEAVAILABLE: result = statusCacheAvailable - else: result = statusUnsupported - -proc addRef(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - inc(self.objectRefCount) - result = self.objectRefCount - -proc release(self: PIBindStatusCallback): ULONG {.gcsafe, stdcall.} = - dec(self.objectRefCount) - result = self.objectRefCount - -proc queryInterface(self: PIBindStatusCallback, riid: ptr UUID, - pvObject: ptr pointer): HRESULT {.gcsafe,stdcall.} = - pvObject[] = nil - - if riid[] == IID_IUnknown: - pvObject[] = cast[pointer](self) - elif riid[] == IID_IBindStatusCallback: - pvObject[] = cast[pointer](self) - - if not isNil(pvObject[]): - discard addRef(self) - result = S_OK - else: - result = E_NOINTERFACE - -proc onStartBinding(self: PIBindStatusCallback, dwReserved: DWORD, - pib: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc getPriority(self: PIBindStatusCallback, - pnPriority: ptr LONG): HRESULT {.gcsafe, stdcall.} = - result = E_NOTIMPL - -proc onLowResource(self: PIBindStatusCallback, - dwReserved: DWORD): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onStopBinding(self: PIBindStatusCallback, - hresult: HRESULT, szError: LPWSTR): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc getBindInfo(self: PIBindStatusCallback, - grfBINDF: ptr DWORD, pbindinfo: ptr BINDINFO): HRESULT - {.gcsafe, stdcall.} = - var cbSize = pbindinfo.cbSize - zeroMem(cast[pointer](pbindinfo), cbSize) - pbindinfo.cbSize = cbSize - grfBINDF[] = self.binfoFlags - result = S_OK - -proc onDataAvailable(self: PIBindStatusCallback, - grfBSCF: DWORD, dwSize: DWORD, pformatetc: pointer, - pstgmed: pointer): HRESULT {.gcsafe, stdcall.} = - result = S_OK - -proc onObjectAvailable(self: PIBindStatusCallback, - riid: REFIID, punk: pointer): HRESULT - {.gcsafe, stdcall.} = - result = S_OK - -proc onProgress(self: PIBindStatusCallback, - ulProgress: ULONG, ulProgressMax: ULONG, ulStatusCode: ULONG, - szStatusText: LPWSTR): HRESULT {.gcsafe, stdcall.} = - var message: string - if optUseProgressCallback in self.options: - if not isNil(szStatusText): - message = $szStatusText - else: - message = "" - self.progressCallback(getStatus(ulStatusCode), uint(ulProgress), - uint(ulProgressMax), message) - result = S_OK - -proc newBindStatusCallback(): IBindStatusCallback = - result = IBindStatusCallback() - result.vtable = cast[ptr IBindStatusCallbackVTable]( - alloc0(sizeof(IBindStatusCallbackVTable)) - ) - result.vtable.QueryInterface = queryInterface - result.vtable.AddRef = addRef - result.vtable.Release = release - result.vtable.OnStartBinding = onStartBinding - result.vtable.GetPriority = getPriority - result.vtable.OnLowResource = onLowResource - result.vtable.OnStopBinding = onStopBinding - result.vtable.GetBindInfo = getBindInfo - result.vtable.OnDataAvailable = onDataAvailable - result.vtable.OnObjectAvailable = onObjectAvailable - result.vtable.OnProgress = onProgress - result.objectRefCount = 1 - -proc freeBindStatusCallback(v: var IBindStatusCallback) = - dealloc(v.vtable) - -proc downloadToFile*(szUrl: string, szFileName: string, - options: set[DownloadOptions] = {}, - progresscb: DownloadProgressCallback = nil) = - ## Downloads from URL specified in ``szUrl`` to local filesystem path - ## specified in ``szFileName``. - ## - ## szUrl - ## URL to download, international names are supported. - ## szFileName - ## Destination path for downloading resource. - ## options - ## Downloading options. Currently only 2 options supported. - ## progresscb - ## Callback procedure, which will be called throughout the download - ## process, indicating status and progress. - ## - ## Available downloading options: - ## - ## optUseCache - ## Try to use Windows cache when downloading. - ## optIgnoreSecurity - ## Ignore HTTPS security problems, e.g. self-signed HTTPS certificate. - ## - var bszUrl = szUrl.toBstring() - var bszFile = szFileName.toBstring() - var bstatus = newBindStatusCallback() - - bstatus.options = {} - - if optUseCache notin options: - bstatus.options.incl(optUseCache) - let res = DeleteUrlCacheEntry(bszUrl) - if res == 0: - let err = osLastError() - if err.int notin {ERROR_ACCESS_DENIED, ERROR_FILE_NOT_FOUND}: - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(err) - - bstatus.binfoFlags = BINDF_GETNEWESTVERSION or BINDF_RESYNCHRONIZE or - BINDF_PRAGMA_NO_CACHE or BINDF_NO_UI or - BINDF_SILENTOPERATION - - if optIgnoreSecurity in options: - bstatus.binfoFlags = bstatus.binfoFlags or BINDF_IGNORESECURITYPROBLEM - - if not isNil(progresscb): - bstatus.options.incl(optUseProgressCallback) - bstatus.progressCallback = progresscb - - let res = URLDownloadToFile(nil, bszUrl, bszFile, 0, addr bstatus) - if FAILED(res): - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - raiseOsError(OSErrorCode(res)) - - freeBindStatusCallback(bstatus) - freeBstring(bszUrl) - freeBstring(bszFile) - -when isMainModule: - proc progress(status: DownloadStatus, progress: uint, progressMax: uint, - message: string) {.gcsafe.} = - const downset: set[DownloadStatus] = {statusBeginDownloading, - statusDownloading, statusEndDownloading} - if status in downset: - var message = "Downloaded " & $progress & " of " & $progressMax & "\c" - stdout.write(message) - else: - echo "Status [" & $status & "] message = [" & $message & "]" - - downloadToFile("https://nim-lang.org/download/mingw64.7z", - "test.zip", {optUseCache}, progress) diff --git a/tools/vccexe/vccexe.nim b/tools/vccexe/vccexe.nim index abe68c0a0..2a43f7422 100644 --- a/tools/vccexe/vccexe.nim +++ b/tools/vccexe/vccexe.nim @@ -41,6 +41,7 @@ const platformPrefix = "--platform" sdktypePrefix = "--sdktype" sdkversionPrefix = "--sdkversion" + vctoolsetPrefix = "--vctoolset" verbosePrefix = "--verbose" vccversionSepIdx = vccversionPrefix.len @@ -49,6 +50,7 @@ const platformSepIdx = platformPrefix.len sdktypeSepIdx = sdktypePrefix.len sdkversionSepIdx = sdkversionPrefix.len + vctoolsetSepIdx = vctoolsetPrefix.len vcvarsallDefaultPath = "vcvarsall.bat" @@ -97,6 +99,8 @@ Options: "8.1" to use the windows 8.1 SDK --verbose Echoes the command line for loading the Developer Command Prompt and the command line passed on to the secondary command. + --vctoolset Optionally specifies the Visual Studio compiler toolset to use. + By default, the environment is set to use the current Visual Studio compiler toolset. Other command line arguments are passed on to the secondary command specified by --command or to the @@ -108,7 +112,7 @@ proc parseVccexeCmdLine(argseq: seq[string], vccversionArg: var seq[string], printPathArg: var bool, vcvarsallArg: var string, commandArg: var string, noCommandArg: var bool, platformArg: var VccArch, sdkTypeArg: var VccPlatformType, - sdkVersionArg: var string, verboseArg: var bool, + sdkVersionArg: var string, vctoolsetArg: var string, verboseArg: var bool, clArgs: var seq[string]) = ## Cannot use usual command-line argument parser here ## Since vccexe command-line arguments are intermingled @@ -125,7 +129,7 @@ proc parseVccexeCmdLine(argseq: seq[string], responseargs = parseCmdLine(responsecontent) parseVccexeCmdLine(responseargs, vccversionArg, printPathArg, vcvarsallArg, commandArg, noCommandArg, platformArg, sdkTypeArg, - sdkVersionArg, verboseArg, clArgs) + sdkVersionArg, vctoolsetArg, verboseArg, clArgs) elif wargv.startsWith(vccversionPrefix): # Check for vccversion vccversionArg.add(wargv.substr(vccversionSepIdx + 1)) elif wargv.cmpIgnoreCase(printPathPrefix) == 0: # Check for printPath @@ -142,6 +146,8 @@ proc parseVccexeCmdLine(argseq: seq[string], sdkTypeArg = parseEnum[VccPlatformType](wargv.substr(sdktypeSepIdx + 1)) elif wargv.startsWith(sdkversionPrefix): # Check for sdkversion sdkVersionArg = wargv.substr(sdkversionSepIdx + 1) + elif wargv.startsWith(vctoolsetPrefix): # Check for vctoolset + vctoolsetArg = wargv.substr(vctoolsetSepIdx + 1) elif wargv.startsWith(verbosePrefix): verboseArg = true else: # Regular cl.exe argument -> store for final cl.exe invocation @@ -158,13 +164,14 @@ when isMainModule: var platformArg: VccArch var sdkTypeArg: VccPlatformType var sdkVersionArg: string + var vctoolsetArg: string var verboseArg: bool = false var clArgs: seq[string] = @[] let wrapperArgs = commandLineParams() parseVccexeCmdLine(wrapperArgs, vccversionArg, printPathArg, vcvarsallArg, - commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, + commandArg, noCommandArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg, clArgs) @@ -195,7 +202,7 @@ when isMainModule: echo "$1: $2" % [head, vcvarsallArg] # Call vcvarsall to get the appropriate VCC process environment - var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, verboseArg) + var vcvars = vccVarsAll(vcvarsallArg, platformArg, sdkTypeArg, sdkVersionArg, vctoolsetArg, verboseArg) if vcvars != nil: for vccEnvKey, vccEnvVal in vcvars: putEnv(vccEnvKey, vccEnvVal) @@ -204,6 +211,11 @@ when isMainModule: if verboseArg: vccOptions.incl poEchoCmd + let currentDir = getCurrentDir() + for arg in clArgs.mitems: + if fileExists(arg): + arg = relativePath(arg, currentDir) + # Default to the cl.exe command if no secondary command was specified if commandArg.len < 1: commandArg = "cl.exe" diff --git a/tools/vccexe/vcvarsall.nim b/tools/vccexe/vcvarsall.nim index 29d13cc7e..73b103e3c 100644 --- a/tools/vccexe/vcvarsall.nim +++ b/tools/vccexe/vcvarsall.nim @@ -33,7 +33,7 @@ type vccplatUWP = "uwp", ## Universal Windows Platform (UWP) Application vccplatOneCore = "onecore" # Undocumented platform type in the Windows SDK, probably XBox One SDK platform type. -proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version: string = "", verbose: bool = false): StringTableRef = +proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type: VccPlatformType = vccplatEmpty, sdk_version, vctoolset: string = "", verbose: bool = false): StringTableRef = ## Returns a string table containing the proper process environment to successfully execute VCC compile commands for the specified SDK version, CPU architecture and platform type. ## ## path @@ -44,6 +44,8 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type ## The compile target Platform Type. Defaults to the Windows Desktop platform, i.e. a regular Windows executable binary. ## sdk_version ## The Windows SDK version to use. + ## vctoolset + ## Visual Studio compiler toolset to use. ## verbose ## Echo the command-line passed on to the system to load the VCC environment. Defaults to `false`. @@ -63,6 +65,9 @@ proc vccVarsAll*(path: string, arch: VccArch = vccarchUnspecified, platform_type if sdk_version.len > 0: args.add(sdk_version) + + if vctoolset.len > 0: + args.add("-vcvars_ver="&vctoolset) let argStr = args.join " " diff --git a/tools/website.nimf b/tools/website.nimf deleted file mode 100644 index bde84b32f..000000000 --- a/tools/website.nimf +++ /dev/null @@ -1,266 +0,0 @@ -#? stdtmpl | standard -#proc generateHTMLPage(c: var TConfigData, currentTab, title, content, rss, -# rootDir = ""): string = -# result = "" -<!DOCTYPE html> -<html> - <head> - <meta http-equiv="content-type" content="text/html; charset=utf-8"> - <title>${title} - $c.projectTitle</title> - <link rel="stylesheet" type="text/css" href="${rootDir}assets/style.css?t=2320" /> - <link rel="shortcut icon" href="${rootDir}assets/images/favicon.ico"> - #if len(rss) > 0: - <link href="$rss" title="Recent changes" type="application/atom+xml" rel="alternate"> - #end if - </head> - <body> - <div id="bountysource"> - <a href="https://salt.bountysource.com/teams/nim"> - <div class="page-layout" style="padding: 2pt 2pt 2pt 30pt"> - <img src="${rootDir}assets/bountysource/bountysource.png" style="width: 20px; float: left;"> - <span style="margin-left: 10pt; float: left; margin-top: 2pt;">Fund Nim and help us develop it further!</span> - <img src="https://api.bountysource.com/badge/team?team_id=19072&style=raised" style="margin-top: 2pt; margin-left: 10pt"/> - </div> - </a> - </div> - - <header id="head"> - <div class="page-layout tall"> - <div id="head-logo"></div> - <a id="head-logo-link" href="${rootDir}index.html"></a> - <nav id="head-links"> - #for i in 0.. c.tabs.len-1: - # let t = c.tabs[i].val - # if t != "index" and t != "community" and t != "news": - # let name = c.tabs[i].key - # if currentTab == t: - <a class="active" - # else: - <a - # end if - # if t.contains('.'): - href="${t}" title = "$c.projectName - $name">$name</a> - # else: - href="${rootDir}${t}.html" title = "$c.projectName - $name">$name</a> - # end if - # end if - #end for - </nav> - </div> - </header> - -# if currentTab == "index": - <section id="neck" class="home"> -# else: - <section id="neck"> -# end - <div class="page-layout tall"> - <div id="glow-arrow"></div> - -# if currentTab == "index": - <div id="slideshow"> - <!-- slides --> - <div id="slide0" class="active codeslide2"> - <div> - <h2>Nim is simple..</h2> -<pre> -<span class="cmt"># compute average line length</span> -<span class="kwd">var</span> -<span class="tab"> </span>sum = <span class="val">0</span> -<span class="tab end"> </span>count = <span class="val">0</span> - -<span class="kwd">for</span> line <span class="kwd">in</span> stdin.lines: -<span class="tab"> </span>sum += line.len -<span class="tab end"> </span>count += <span class="val">1</span> - -echo(<span class="val">"Average line length: "</span>, - <span class="kwd">if</span> count > <span class="val">0</span>: sum / count <span class="kwd">else</span>: <span class="val">0</span>) -</pre> - </div> - <div> - <h2>..and type safe...</h2> -<pre> -<span class="cmt"># create and greet someone</span> -<span class="kwd">type</span> <span class="def">Person</span> = <span class="kwd">object</span> -<span class="tab"> </span>name: <span class="typ">string</span> -<span class="tab end"> </span>age: <span class="typ">int</span> - -<span class="kwd">proc</span> <span class="def">greet</span>(p: <span class="typ">Person</span>) = -<span class="tab"> </span>echo <span class="val">"Hi, I'm "</span>, p.name, <span class="val">"."</span> -<span class="tab end"> </span>echo <span class="val">"I am "</span>, p.age, <span class="val">" years old."</span> - -<span class="kwd">let</span> p = <span class="typ">Person</span>(name:<span class="val">"Jon"</span>, age:<span class="val">18</span>) -p.greet() <span class="cmt"># or greet(p)</span> -</pre> - </div> - </div> - <div id="slide1" class="codeslide3"> - <div> - <h2>C FFI is easy in Nim..</h2> -<pre> -<span class="cmt"># declare a C procedure..</span> -<span class="kwd">proc</span> <span class="def">unsafeScanf</span>(f: <span class="typ">File</span>, s: <span class="typ">cstring</span>) -<span class="tab"> </span>{.varargs, -<span class="tab"> </span>importc: <span class="val">"fscanf"</span>, -<span class="tab end"> </span>header: <span class="val">"<stdio.h>"</span>.} - -<span class="cmt"># ..and use it...</span> -<span class="kwd">var</span> x: <span class="typ">cint</span> -stdin.unsafeScanf(<span class="val">"%d"</span>, <span class="kwd">addr</span> x) -</pre> - <p><span class="desc"><b>Compile and run with:</b><br> $ nim c -r example.nim</span></p> - </div> - <div> - <h2>..and DSLs are too...</h2> -<pre> -<span class="cmt"># a simple html server</span> -<span class="kwd">import</span> - jester, asyncdispatch, htmlgen - -<span class="kwd">routes</span>: -<span class="tab"> </span><span class="kwd">get</span> <span class="val">"/"</span>: -<span class="tab end"> <span class="tab end"> </span></span><span class="kwd">resp</span> h1(<span class="val">"Hello world"</span>) - -runForever() -</pre> - <p><span class="desc"><b>View in browser at:</b><br> localhost:5000</span></p> - </div> - </div> - <div id="slide2" class="niaslide"> - <a href="news/e030_nim_in_action_in_production.html"> - <img src="${rootDir}assets/niminaction/banner2.png" alt="A printed copy of Nim in Action should be available in March 2017!"/> - </a> - </div> - <div id="slide3" class="niaslide"> - <a href="sponsors.html"> - <img src="${rootDir}assets/bountysource/meet_sponsors.png" alt="Meet our BountySource sponsors!"/> - </a> - </div> - </div> - <div id="slideshow-nav"> - <div id="slideControl0" onclick="slideshow_click(0)" class="active"></div> - <div id="slideControl1" onclick="slideshow_click(1)"></div> - <div id="slideControl2" onclick="slideshow_click(2)"></div> - <div id="slideControl3" onclick="slideshow_click(3)"></div> - </div> -# end - <aside id="sidebar"> - -# if len(c.links) > 0: - <h3>More Links</h3> - <div id="sidebar-links"> -# for i in 0..c.links.len-1: - <a href="${c.links[i].val}" id="${c.links[i].id}">${c.links[i].key}</a> -# end for - </div> -# end if -# if len(c.ticker) > 0: - <h3 class="blue">Latest News</h3> - <div id="sidebar-news"> - ${c.ticker % rootDir} - </div> -# end if - </aside> - </div> - </section> - - <section id="body"> - <div id="body-border"></div> - <div id="glow-line"></div> - <div class="page-layout"> - <article id="content" class="page"> - $content - </article> - </div> - </section> - - <!--- #foot ---> - <footer id="foot" class="home"> - <div class="page-layout tall"> - <div id="foot-links"> - <div> - <h4>Documentation</h4> - <a href="${rootDir}documentation.html">Stable Documentation</a> - <a href="${rootDir}learn.html">Learning Resources</a> - <!-- <a href="">Development Documentation</a> --> - <a href="https://github.com/nim-lang/nim">Issues & Requests</a> - </div> - <div> - <h4>Community</h4> - <a href="https://forum.nim-lang.org">User Forum</a> - <a href="http://webchat.freenode.net/?channels=nim">Online IRC</a> - <a href="https://irclogs.nim-lang.org/">IRC Logs</a> - </div> - </div> - <div id="foot-legal"> - <h4>Written in Nim - Powered by <a href="https://github.com/dom96/jester">Jester</a></h4> - Web Design by <a href="http://reign-studios.net/philipwitte/">Philip Witte</a> & <a href="http://picheta.me/">Dominik Picheta</a><br> - Copyright © 2017 - <a href="https://nim-lang.org/blog/">Andreas Rumpf</a> & <a href="https://github.com/nim-lang/nim/graphs/contributors">Contributors</a> - </div> - </div> - </footer> - -# if currentTab == "index": - <script src="${rootDir}assets/index.js"></script> -# end if -# if c.gaId.len != 0: - <script> - (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ - (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), - m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) - })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - - ga('create', '${c.gaId}', 'nim-lang.org'); - ga('send', 'pageview'); - </script> -# end if -</body> -</html> -#end proc -# -# -#proc generateSponsors(sponsors: seq[Sponsor]): string = -#result = "" -#for sponsor in sponsors: - <dt class="level-${sponsor.level}"> - #if sponsor.url.len > 0: - <a href="${sponsor.url}" target="_blank">${sponsor.name}</a> - #else: - ${sponsor.name} - #end if - </dt> - <dd class="logo"> - #if sponsor.logo.len > 0: - <a href="${sponsor.url}" target="_blank"> - <img alt="${sponsor.name}'s logo" src="${sponsor.logo}"/> - </a> - #end if - </dd> - <dd class="this_month"> - Donated <b>$$${sponsor.thisMonth}</b> this month - </dd> - <dd class="legend"> - Donated $$${sponsor.allTime} in total since ${sponsor.since} - </dd> -#end for -#end proc -#proc generateSponsorsPage(activeSponsors, inactiveSponsors: seq[Sponsor]): string = -#result = "" -<h1 id="our-current-sponsors">Our Current Sponsors</h1> -<p>This section lists the companies and individuals that are, very kindly, contributing a -monthly amount to help sustain Nim's development. For more details take a -look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> -<p class="lastUpdate">Last updated: ${getTime().utc().format("dd/MM/yyyy")}</p> -<dl> -${generateSponsors(activeSponsors)} -</dl> -# -<h1 id="our-past-sponsors">Our Past Sponsors</h1> -<p>This section lists the companies and individuals that have contributed -money in the past to help sustain Nim's development. For more details take a -look at the <a href="https://salt.bountysource.com/teams/nim">Bountysource campaign</a>.</p> -<dl> -${generateSponsors(inactiveSponsors)} -</dl> -# -#end proc |