diff options
Diffstat (limited to 'tools/nimsuggest')
| -rw-r--r-- | tools/nimsuggest/nimsuggest.nim | 475 | ||||
| -rw-r--r-- | tools/nimsuggest/nimsuggest.nim.cfg | 16 | ||||
| -rw-r--r-- | tools/nimsuggest/nimsuggest.nimble | 11 | ||||
| -rw-r--r-- | tools/nimsuggest/sexp.nim | 697 | ||||
| -rw-r--r-- | tools/nimsuggest/tester.nim | 182 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/dep_v1.nim | 8 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/dep_v2.nim | 9 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tdef1.nim | 16 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tdot1.nim | 14 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tdot2.nim | 29 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tdot3.nim | 27 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tinclude.nim | 7 | ||||
| -rw-r--r-- | tools/nimsuggest/tests/tstrutils.nim | 9 |
13 files changed, 1500 insertions, 0 deletions
diff --git a/tools/nimsuggest/nimsuggest.nim b/tools/nimsuggest/nimsuggest.nim new file mode 100644 index 000000000..b5e7b282f --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nim @@ -0,0 +1,475 @@ +# +# +# The Nim Compiler +# (c) Copyright 2016 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Nimsuggest is a tool that helps to give editors IDE like capabilities. + +import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp +# Do NOT import suggest. It will lead to wierd bugs with +# suggestionResultHook, because suggest.nim is included by sigmatch. +# So we import that one instead. +import compiler/options, compiler/commands, compiler/modules, compiler/sem, + compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf, + compiler/extccomp, compiler/condsyms, compiler/lists, + compiler/sigmatch, compiler/ast, compiler/scriptconfig, + compiler/idents, compiler/modulegraphs + +when defined(windows): + import winlean +else: + import posix + +const DummyEof = "!EOF!" +const Usage = """ +Nimsuggest - Tool to give every editor IDE like capabilities for Nim +Usage: + nimsuggest [options] projectfile.nim + +Options: + --port:PORT port, by default 6000 + --address:HOST binds to that address, by default "" + --stdin read commands from stdin and write results to + stdout instead of using sockets + --epc use emacs epc mode + --debug enable debug output + --log enable verbose logging to nimsuggest.log file + --v2 use version 2 of the protocol; more features and + much faster + --tester implies --v2 and --stdin and outputs a line + '""" & DummyEof & """' for the tester + +The server then listens to the connection and takes line-based commands. + +In addition, all command line options of Nim that do not affect code generation +are supported. +""" +type + Mode = enum mstdin, mtcp, mepc + +var + gPort = 6000.Port + gAddress = "" + gMode: Mode + gEmitEof: bool # whether we write '!EOF!' dummy lines + gLogging = false + +const + seps = {':', ';', ' ', '\t'} + Help = "usage: sug|con|def|use|dus|chk|mod|highlight|outline file.nim[;dirtyfile.nim]:line:col\n" & + "type 'quit' to quit\n" & + "type 'debug' to toggle debug mode on/off\n" & + "type 'terse' to toggle terse mode on/off" + +type + EUnexpectedCommand = object of Exception + +proc logStr(line: string) = + var f: File + if open(f, getHomeDir() / "nimsuggest.log", fmAppend): + f.writeLine(line) + f.close() + +proc parseQuoted(cmd: string; outp: var string; start: int): int = + var i = start + i += skipWhitespace(cmd, i) + if cmd[i] == '"': + i += parseUntil(cmd, outp, '"', i+1)+2 + else: + i += parseUntil(cmd, outp, seps, i) + result = i + +proc sexp(s: IdeCmd|TSymKind): SexpNode = sexp($s) + +proc sexp(s: Suggest): SexpNode = + # If you change the order here, make sure to change it over in + # nim-mode.el too. + result = convertSexp([ + s.section, + s.symkind, + s.qualifiedPath.map(newSString), + s.filePath, + s.forth, + s.line, + s.column, + s.doc + ]) + +proc sexp(s: seq[Suggest]): SexpNode = + result = newSList() + for sug in s: + result.add(sexp(sug)) + +proc listEpc(): SexpNode = + # This function is called from Emacs to show available options. + let + argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol)) + docstring = sexp("line starts at 1, column at 0, dirtyfile is optional") + result = newSList() + for command in ["sug", "con", "def", "use", "dus", "chk", "mod"]: + let + cmd = sexp(command) + methodDesc = newSList() + methodDesc.add(cmd) + methodDesc.add(argspecs) + methodDesc.add(docstring) + result.add(methodDesc) + +proc findNode(n: PNode): PSym = + #echo "checking node ", n.info + if n.kind == nkSym: + if isTracked(n.info, n.sym.name.s.len): return n.sym + else: + for i in 0 ..< safeLen(n): + let res = n.sons[i].findNode + if res != nil: return res + +proc symFromInfo(graph: ModuleGraph; gTrackPos: TLineInfo): PSym = + let m = graph.getModule(gTrackPos.fileIndex) + #echo m.isNil, " I knew it ", gTrackPos.fileIndex + if m != nil and m.ast != nil: + result = m.ast.findNode + +proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int; + graph: ModuleGraph; cache: IdentCache) = + if gLogging: + logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]") + gIdeCmd = cmd + if cmd == ideUse and suggestVersion != 2: + graph.resetAllModules() + var isKnownFile = true + let dirtyIdx = file.fileInfoIdx(isKnownFile) + + if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile) + else: msgs.setDirtyFile(dirtyIdx, nil) + + gTrackPos = newLineInfo(dirtyIdx, line, col) + gErrorCounter = 0 + if suggestVersion < 2: + usageSym = nil + if not isKnownFile: + graph.compileProject(cache) + if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and + dirtyfile.len == 0: + discard "no need to recompile anything" + else: + let modIdx = graph.parentModule(dirtyIdx) + graph.markDirty dirtyIdx + graph.markClientsDirty dirtyIdx + if gIdeCmd != ideMod: + graph.compileProject(cache, modIdx) + if gIdeCmd in {ideUse, ideDus}: + let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: usageSym + if u != nil: + listUsages(u) + else: + localError(gTrackPos, "found no symbol at this position " & $gTrackPos) + +proc executeEpc(cmd: IdeCmd, args: SexpNode; + graph: ModuleGraph; cache: IdentCache) = + let + file = args[0].getStr + line = args[1].getNum + column = args[2].getNum + var dirtyfile = "" + if len(args) > 3: + dirtyfile = args[3].getStr(nil) + execute(cmd, file, dirtyfile, int(line), int(column), graph, cache) + +proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string, + return_symbol = "return") = + let response = $convertSexp([newSSymbol(return_symbol), uid, s]) + socket.send(toHex(len(response), 6)) + socket.send(response) + +template sendEpc(results: typed, tdef, hook: untyped) = + hook = proc (s: tdef) = + results.add( + # Put newlines to parse output by flycheck-nim.el + when results is string: s & "\n" + else: s + ) + + executeEpc(gIdeCmd, args, graph, cache) + let res = sexp(results) + if gLogging: + logStr($res) + returnEPC(client, uid, res) + +template checkSanity(client, sizeHex, size, messageBuffer: typed) = + if client.recv(sizeHex, 6) != 6: + raise newException(ValueError, "didn't get all the hexbytes") + if parseHex(sizeHex, size) == 0: + raise newException(ValueError, "invalid size hex: " & $sizeHex) + if client.recv(messageBuffer, size) != size: + raise newException(ValueError, "didn't get all the bytes") + +template setVerbosity(level: typed) = + gVerbosity = level + gNotes = NotesVerbosity[gVerbosity] + +proc connectToNextFreePort(server: Socket, host: string): Port = + server.bindaddr(Port(0), host) + let (_, port) = server.getLocalAddr + result = port + +proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) = + template toggle(sw) = + if sw in gGlobalOptions: + excl(gGlobalOptions, sw) + else: + incl(gGlobalOptions, sw) + return + + template err() = + echo Help + return + + var opc = "" + var i = parseIdent(cmd, opc, 0) + case opc.normalize + of "sug": gIdeCmd = ideSug + of "con": gIdeCmd = ideCon + of "def": gIdeCmd = ideDef + of "use": gIdeCmd = ideUse + of "dus": gIdeCmd = ideDus + of "mod": gIdeCmd = ideMod + of "chk": + gIdeCmd = ideChk + incl(gGlobalOptions, optIdeDebug) + of "highlight": gIdeCmd = ideHighlight + of "outline": gIdeCmd = ideOutline + of "quit": quit() + of "debug": toggle optIdeDebug + of "terse": toggle optIdeTerse + else: err() + var dirtyfile = "" + var orig = "" + i = parseQuoted(cmd, orig, i) + if cmd[i] == ';': + i = parseQuoted(cmd, dirtyfile, i+1) + i += skipWhile(cmd, seps, i) + var line = -1 + var col = 0 + i += parseInt(cmd, line, i) + i += skipWhile(cmd, seps, i) + i += parseInt(cmd, col, i) + + execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache) + +proc serveStdin(graph: ModuleGraph; cache: IdentCache) = + if gEmitEof: + echo DummyEof + while true: + let line = readLine(stdin) + parseCmdLine line, graph, cache + echo DummyEof + flushFile(stdout) + else: + echo Help + var line = "" + while readLineFromStdin("> ", line): + parseCmdLine line, graph, cache + echo "" + flushFile(stdout) + +proc serveTcp(graph: ModuleGraph; cache: IdentCache) = + var server = newSocket() + server.bindAddr(gPort, gAddress) + var inp = "".TaintedString + server.listen() + + while true: + var stdoutSocket = newSocket() + msgs.writelnHook = proc (line: string) = + stdoutSocket.send(line & "\c\L") + + accept(server, stdoutSocket) + + stdoutSocket.readLine(inp) + parseCmdLine inp.string, graph, cache + + stdoutSocket.send("\c\L") + stdoutSocket.close() + +proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) = + var client = newSocket() + # Wait for connection + accept(server, client) + if gLogging: + var it = searchPaths.head + while it != nil: + logStr(PStrEntry(it).data) + it = it.next + msgs.writelnHook = proc (line: string) = logStr(line) + + while true: + var + sizeHex = "" + size = 0 + messageBuffer = "" + checkSanity(client, sizeHex, size, messageBuffer) + let + message = parseSexp($messageBuffer) + epcAPI = message[0].getSymbol + case epcAPI: + of "call": + let + uid = message[1].getNum + args = message[3] + + gIdeCmd = parseIdeCmd(message[2].getSymbol) + case gIdeCmd + of ideChk: + setVerbosity(1) + # Use full path because other emacs plugins depends it + gListFullPaths = true + incl(gGlobalOptions, optIdeDebug) + var hints_or_errors = "" + sendEpc(hints_or_errors, string, msgs.writelnHook) + of ideSug, ideCon, ideDef, ideUse, ideDus, ideOutline, ideHighlight: + setVerbosity(0) + var suggests: seq[Suggest] = @[] + sendEpc(suggests, Suggest, suggestionResultHook) + else: discard + of "methods": + returnEpc(client, message[1].getNum, listEPC()) + of "epc-error": + stderr.writeline("recieved epc error: " & $messageBuffer) + raise newException(IOError, "epc error") + else: + let errMessage = case epcAPI + of "return", "return-error": + "no return expected" + else: + "unexpected call: " & epcAPI + raise newException(EUnexpectedCommand, errMessage) + +proc mainCommand(graph: ModuleGraph; cache: IdentCache) = + clearPasses() + registerPass verbosePass + registerPass semPass + gCmd = cmdIdeTools + incl gGlobalOptions, optCaasEnabled + isServing = true + wantMainModule() + appendStr(searchPaths, options.libpath) + #if gProjectFull.len != 0: + # current path is always looked first for modules + # prependStr(searchPaths, gProjectPath) + + # do not stop after the first error: + msgs.gErrorMax = high(int) + + case gMode + of mstdin: + compileProject(graph, cache) + #modules.gFuzzyGraphChecking = false + serveStdin(graph, cache) + of mtcp: + # until somebody accepted the connection, produce no output (logging is too + # slow for big projects): + msgs.writelnHook = proc (msg: string) = discard + compileProject(graph, cache) + #modules.gFuzzyGraphChecking = false + serveTcp(graph, cache) + of mepc: + var server = newSocket() + let port = connectToNextFreePort(server, "localhost") + server.listen() + echo port + compileProject(graph, cache) + serveEpc(server, graph, cache) + +proc processCmdLine*(pass: TCmdLinePass, cmd: string) = + var p = parseopt.initOptParser(cmd) + while true: + parseopt.next(p) + case p.kind + of cmdEnd: break + of cmdLongoption, cmdShortOption: + case p.key.normalize + of "port": + gPort = parseInt(p.val).Port + gMode = mtcp + of "address": + gAddress = p.val + gMode = mtcp + of "stdin": gMode = mstdin + of "epc": + gMode = mepc + gVerbosity = 0 # Port number gotta be first. + of "debug": + incl(gGlobalOptions, optIdeDebug) + of "v2": + suggestVersion = 2 + of "tester": + suggestVersion = 2 + gMode = mstdin + gEmitEof = true + of "log": + gLogging = true + else: processSwitch(pass, p) + of cmdArgument: + options.gProjectName = unixToNativePath(p.key) + # if processArgument(pass, p, argsCount): break + +proc handleCmdLine(cache: IdentCache) = + if paramCount() == 0: + stdout.writeline(Usage) + else: + processCmdLine(passCmd1, "") + if gMode != mstdin: + msgs.writelnHook = proc (msg: string) = discard + if gProjectName != "": + try: + gProjectFull = canonicalizePath(gProjectName) + except OSError: + gProjectFull = gProjectName + var p = splitFile(gProjectFull) + gProjectPath = canonicalizePath p.dir + gProjectName = p.name + else: + gProjectPath = canonicalizePath getCurrentDir() + + # Find Nim's prefix dir. + let binaryPath = findExe("nim") + if binaryPath == "": + raise newException(IOError, + "Cannot find Nim standard library: Nim compiler not in PATH") + gPrefixDir = binaryPath.splitPath().head.parentDir() + #msgs.writelnHook = proc (line: string) = logStr(line) + + loadConfigs(DefaultConfig, cache) # load all config files + # now process command line arguments again, because some options in the + # command line can overwite the config file's settings + options.command = "nimsuggest" + let scriptFile = gProjectFull.changeFileExt("nims") + if fileExists(scriptFile): + runNimScript(cache, scriptFile, freshDefines=false) + # 'nim foo.nims' means to just run the NimScript file and do nothing more: + if scriptFile == gProjectFull: return + elif fileExists(gProjectPath / "config.nims"): + # directory wide NimScript file + runNimScript(cache, gProjectPath / "config.nims", freshDefines=false) + + extccomp.initVars() + processCmdLine(passCmd2, "") + + let graph = newModuleGraph() + graph.suggestMode = true + mainCommand(graph, cache) + +when false: + proc quitCalled() {.noconv.} = + writeStackTrace() + + addQuitProc(quitCalled) + +condsyms.initDefines() +defineSymbol "nimsuggest" +handleCmdline(newIdentCache()) diff --git a/tools/nimsuggest/nimsuggest.nim.cfg b/tools/nimsuggest/nimsuggest.nim.cfg new file mode 100644 index 000000000..949bd18e8 --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nim.cfg @@ -0,0 +1,16 @@ +# Special configuration file for the Nim project + +gc:markAndSweep + +hint[XDeclaredButNotUsed]:off + +path:"$lib/packages/docutils" + +define:useStdoutAsStdmsg +define:nimsuggest + +#cs:partial +#define:useNodeIds +#define:booting +#define:noDocgen +--path:"$nim" diff --git a/tools/nimsuggest/nimsuggest.nimble b/tools/nimsuggest/nimsuggest.nimble new file mode 100644 index 000000000..3651e12bd --- /dev/null +++ b/tools/nimsuggest/nimsuggest.nimble @@ -0,0 +1,11 @@ +[Package] +name = "nimsuggest" +version = "0.1.0" +author = "Andreas Rumpf" +description = "Tool for providing auto completion data for Nim source code." +license = "MIT" + +bin = "nimsuggest" + +[Deps] +Requires: "nim >= 0.11.2, compiler#head" diff --git a/tools/nimsuggest/sexp.nim b/tools/nimsuggest/sexp.nim new file mode 100644 index 000000000..cf08111d7 --- /dev/null +++ b/tools/nimsuggest/sexp.nim @@ -0,0 +1,697 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf, Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import + hashes, strutils, lexbase, streams, unicode, macros + +type + SexpEventKind* = enum ## enumeration of all events that may occur when parsing + sexpError, ## an error occurred during parsing + sexpEof, ## end of file reached + sexpString, ## a string literal + sexpSymbol, ## a symbol + sexpInt, ## an integer literal + sexpFloat, ## a float literal + sexpNil, ## the value ``nil`` + sexpDot, ## the dot to separate car/cdr + sexpListStart, ## start of a list: the ``(`` token + sexpListEnd, ## end of a list: the ``)`` token + + TTokKind = enum # must be synchronized with SexpEventKind! + tkError, + tkEof, + tkString, + tkSymbol, + tkInt, + tkFloat, + tkNil, + tkDot, + tkParensLe, + tkParensRi + tkSpace + + SexpError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errParensRiExpected, ## ``)`` expected + errQuoteExpected, ## ``"`` expected + errEofExpected, ## EOF expected + + SexpParser* = object of BaseLexer ## the parser object. + a: string + tok: TTokKind + kind: SexpEventKind + err: SexpError + +const + errorMessages: array[SexpError, string] = [ + "no error", + "invalid token", + "')' expected", + "'\"' or \"'\" expected", + "EOF expected", + ] + tokToStr: array[TTokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "symbol", + "int literal", + "float literal", + "nil", + ".", + "(", ")", "space" + ] + +proc close*(my: var SexpParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: SexpParser): string {.inline.} = + ## returns the character data for the events: ``sexpInt``, ``sexpFloat``, + ## ``sexpString`` + assert(my.kind in {sexpInt, sexpFloat, sexpString}) + result = my.a + +proc getInt*(my: SexpParser): BiggestInt {.inline.} = + ## returns the number for the event: ``sexpInt`` + assert(my.kind == sexpInt) + result = parseBiggestInt(my.a) + +proc getFloat*(my: SexpParser): float {.inline.} = + ## returns the number for the event: ``sexpFloat`` + assert(my.kind == sexpFloat) + result = parseFloat(my.a) + +proc kind*(my: SexpParser): SexpEventKind {.inline.} = + ## returns the current event type for the SEXP parser + result = my.kind + +proc getColumn*(my: SexpParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: SexpParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc errorMsg*(my: SexpParser): string = + ## returns a helpful error message for the event ``sexpError`` + assert(my.kind == sexpError) + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: SexpParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), e & " expected"] + +proc handleHexChar(c: char, x: var int): bool = + result = true # Success + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: result = false # error + +proc parseString(my: var SexpParser): TTokKind = + result = tkString + var pos = my.bufpos + 1 + var buf = my.buf + while true: + case buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + inc(pos) + break + of '\\': + case buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'u': + inc(pos, 2) + var r: int + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + add(my.a, '\L') + else: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc parseNumber(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] == '-': + add(my.a, '-') + inc(pos) + if buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'E', 'e'}: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'+', '-'}: + add(my.a, buf[pos]) + inc(pos) + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseSymbol(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] in IdentStartChars: + while buf[pos] in IdentChars: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok(my: var SexpParser): TTokKind = + setLen(my.a, 0) + case my.buf[my.bufpos] + of '-', '0'..'9': # numbers that start with a . are not parsed + # correctly. + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': #" # gotta fix nim-mode + result = parseString(my) + of '(': + inc(my.bufpos) + result = tkParensLe + of ')': + inc(my.bufpos) + result = tkParensRi + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseSymbol(my) + if my.a == "nil": + result = tkNil + else: + result = tkSymbol + of ' ': + result = tkSpace + inc(my.bufpos) + of '.': + result = tkDot + inc(my.bufpos) + else: + inc(my.bufpos) + result = tkError + my.tok = result + +# ------------- higher level interface --------------------------------------- + +type + SexpNodeKind* = enum ## possible SEXP node types + SNil, + SInt, + SFloat, + SString, + SSymbol, + SList, + SCons + + SexpNode* = ref SexpNodeObj ## SEXP node + SexpNodeObj* {.acyclic.} = object + case kind*: SexpNodeKind + of SString: + str*: string + of SSymbol: + symbol*: string + of SInt: + num*: BiggestInt + of SFloat: + fnum*: float + of SList: + elems*: seq[SexpNode] + of SCons: + car: SexpNode + cdr: SexpNode + of SNil: + discard + + Cons = tuple[car: SexpNode, cdr: SexpNode] + + SexpParsingError* = object of ValueError ## is raised for a SEXP error + +proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} = + ## raises an `ESexpParsingError` exception. + raise newException(SexpParsingError, errorMsgExpected(p, msg)) + +proc newSString*(s: string): SexpNode {.procvar.}= + ## Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc newSStringMove(s: string): SexpNode = + new(result) + result.kind = SString + shallowCopy(result.str, s) + +proc newSInt*(n: BiggestInt): SexpNode {.procvar.} = + ## Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc newSFloat*(n: float): SexpNode {.procvar.} = + ## Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc newSNil*(): SexpNode {.procvar.} = + ## Creates a new `SNil SexpNode`. + new(result) + +proc newSCons*(car, cdr: SexpNode): SexpNode {.procvar.} = + ## Creates a new `SCons SexpNode` + new(result) + result.kind = SCons + result.car = car + result.cdr = cdr + +proc newSList*(): SexpNode {.procvar.} = + ## Creates a new `SList SexpNode` + new(result) + result.kind = SList + result.elems = @[] + +proc newSSymbol*(s: string): SexpNode {.procvar.} = + new(result) + result.kind = SSymbol + result.symbol = s + +proc newSSymbolMove(s: string): SexpNode = + new(result) + result.kind = SSymbol + shallowCopy(result.symbol, s) + +proc getStr*(n: SexpNode, default: string = ""): string = + ## Retrieves the string value of a `SString SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SString``. + if n.kind != SString: return default + else: return n.str + +proc getNum*(n: SexpNode, default: BiggestInt = 0): BiggestInt = + ## Retrieves the int value of a `SInt SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SInt``. + if n.kind != SInt: return default + else: return n.num + +proc getFNum*(n: SexpNode, default: float = 0.0): float = + ## Retrieves the float value of a `SFloat SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SFloat``. + if n.kind != SFloat: return default + else: return n.fnum + +proc getSymbol*(n: SexpNode, default: string = ""): string = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind != SSymbol: return default + else: return n.symbol + +proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SNil: return @[] + elif n.kind != SList: return default + else: return n.elems + +proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons = + ## Retrieves the cons value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SCons: return (n.car, n.cdr) + elif n.kind == SList: return (n.elems[0], n.elems[1]) + else: return defaults + +proc sexp*(s: string): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc sexp*(n: BiggestInt): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc sexp*(n: float): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc sexp*(b: bool): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SSymbol + ## SexpNode` with value t or `SNil SexpNode`. + new(result) + if b: + result.kind = SSymbol + result.symbol = "t" + else: + result.kind = SNil + +proc sexp*(elements: openArray[SexpNode]): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SList SexpNode` + new(result) + result.kind = SList + newSeq(result.elems, elements.len) + for i, p in pairs(elements): result.elems[i] = p + +proc sexp*(s: SexpNode): SexpNode = + result = s + +proc toSexp(x: NimNode): NimNode {.compiletime.} = + case x.kind + of nnkBracket: + result = newNimNode(nnkBracket) + for i in 0 .. <x.len: + result.add(toSexp(x[i])) + + else: + result = x + + result = prefix(result, "sexp") + +macro convertSexp*(x: untyped): untyped = + ## Convert an expression to a SexpNode directly, without having to specify + ## `%` for every element. + result = toSexp(x) + +proc `==`* (a,b: SexpNode): bool = + ## Check two nodes for equality + if a.isNil: + if b.isNil: return true + return false + elif b.isNil or a.kind != b.kind: + return false + else: + return case a.kind + of SString: + a.str == b.str + of SInt: + a.num == b.num + of SFloat: + a.fnum == b.fnum + of SNil: + true + of SList: + a.elems == b.elems + of SSymbol: + a.symbol == b.symbol + of SCons: + a.car == b.car and a.cdr == b.cdr + +proc hash* (n:SexpNode): Hash = + ## Compute the hash for a SEXP node + case n.kind + of SList: + result = hash(n.elems) + of SInt: + result = hash(n.num) + of SFloat: + result = hash(n.fnum) + of SString: + result = hash(n.str) + of SNil: + result = hash(0) + of SSymbol: + result = hash(n.symbol) + of SCons: + result = hash(n.car) !& hash(n.cdr) + +proc len*(n: SexpNode): int = + ## If `n` is a `SList`, it returns the number of elements. + ## If `n` is a `JObject`, it returns the number of pairs. + ## Else it returns 0. + case n.kind + of SList: result = n.elems.len + else: discard + +proc `[]`*(node: SexpNode, index: int): SexpNode = + ## Gets the node at `index` in a List. Result is undefined if `index` + ## is out of bounds + assert(not isNil(node)) + assert(node.kind == SList) + return node.elems[index] + +proc add*(father, child: SexpNode) = + ## Adds `child` to a SList node `father`. + assert father.kind == SList + father.elems.add(child) + +# ------------- pretty printing ---------------------------------------------- + +proc indent(s: var string, i: int) = + s.add(spaces(i)) + +proc newIndent(curr, indent: int, ml: bool): int = + if ml: return curr + indent + else: return indent + +proc nl(s: var string, ml: bool) = + if ml: s.add("\n") + +proc escapeJson*(s: string): string = + ## Converts a string `s` to its JSON representation. + result = newStringOfCap(s.len + s.len shr 3) + result.add("\"") + for x in runes(s): + var r = int(x) + if r >= 32 and r <= 127: + var c = chr(r) + case c + of '"': result.add("\\\"") #" # gotta fix nim-mode + of '\\': result.add("\\\\") + else: result.add(c) + else: + result.add("\\u") + result.add(toHex(r, 4)) + result.add("\"") + +proc copy*(p: SexpNode): SexpNode = + ## Performs a deep copy of `a`. + case p.kind + of SString: + result = newSString(p.str) + of SInt: + result = newSInt(p.num) + of SFloat: + result = newSFloat(p.fnum) + of SNil: + result = newSNil() + of SSymbol: + result = newSSymbol(p.symbol) + of SList: + result = newSList() + for i in items(p.elems): + result.elems.add(copy(i)) + of SCons: + result = newSCons(copy(p.car), copy(p.cdr)) + +proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, + lstArr = false, currIndent = 0) = + case node.kind + of SString: + if lstArr: result.indent(currIndent) + result.add(escapeJson(node.str)) + of SInt: + if lstArr: result.indent(currIndent) + result.add($node.num) + of SFloat: + if lstArr: result.indent(currIndent) + result.add($node.fnum) + of SNil: + if lstArr: result.indent(currIndent) + result.add("nil") + of SSymbol: + if lstArr: result.indent(currIndent) + result.add($node.symbol) + of SList: + if lstArr: result.indent(currIndent) + if len(node.elems) != 0: + result.add("(") + result.nl(ml) + for i in 0..len(node.elems)-1: + if i > 0: + result.add(" ") + result.nl(ml) # New Line + toPretty(result, node.elems[i], indent, ml, + true, newIndent(currIndent, indent, ml)) + result.nl(ml) + result.indent(currIndent) + result.add(")") + else: result.add("nil") + of SCons: + if lstArr: result.indent(currIndent) + result.add("(") + toPretty(result, node.car, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(" . ") + toPretty(result, node.cdr, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(")") + +proc pretty*(node: SexpNode, indent = 2): string = + ## Converts `node` to its Sexp Representation, with indentation and + ## on multiple lines. + result = "" + toPretty(result, node, indent) + +proc `$`*(node: SexpNode): string = + ## Converts `node` to its SEXP Representation on one line. + result = "" + toPretty(result, node, 0, false) + +iterator items*(node: SexpNode): SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. + assert node.kind == SList + for i in items(node.elems): + yield i + +iterator mitems*(node: var SexpNode): var SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. Items can be + ## modified. + assert node.kind == SList + for i in mitems(node.elems): + yield i + +proc eat(p: var SexpParser, tok: TTokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) + +proc parseSexp(p: var SexpParser): SexpNode = + ## Parses SEXP from a SEXP Parser `p`. + case p.tok + of tkString: + # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: + result = newSStringMove(p.a) + p.a = "" + discard getTok(p) + of tkInt: + result = newSInt(parseBiggestInt(p.a)) + discard getTok(p) + of tkFloat: + result = newSFloat(parseFloat(p.a)) + discard getTok(p) + of tkNil: + result = newSNil() + discard getTok(p) + of tkSymbol: + result = newSSymbolMove(p.a) + p.a = "" + discard getTok(p) + of tkParensLe: + result = newSList() + discard getTok(p) + while p.tok notin {tkParensRi, tkDot}: + result.add(parseSexp(p)) + if p.tok != tkSpace: break + discard getTok(p) + if p.tok == tkDot: + eat(p, tkDot) + eat(p, tkSpace) + result.add(parseSexp(p)) + result = newSCons(result[0], result[1]) + eat(p, tkParensRi) + of tkSpace, tkDot, tkError, tkParensRi, tkEof: + raiseParseErr(p, "(") + +proc open*(my: var SexpParser, input: Stream) = + ## initializes the parser with an input stream. + lexbase.open(my, input) + my.kind = sexpError + my.a = "" + +proc parseSexp*(s: Stream): SexpNode = + ## Parses from a buffer `s` into a `SexpNode`. + var p: SexpParser + p.open(s) + discard getTok(p) # read first token + result = p.parseSexp() + p.close() + +proc parseSexp*(buffer: string): SexpNode = + ## Parses Sexp from `buffer`. + result = parseSexp(newStringStream(buffer)) + +when isMainModule: + let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") + assert(testSexp[0].getNum == 1) + assert(testSexp[1][0].getNum == 98) + assert(testSexp[2].getElems == @[]) + assert(testSexp[4].getSymbol == "foobar") + assert(testSexp[5].getStr == "foo") + + let alist = parseSexp("""((1 . 2) (2 . "foo"))""") + assert(alist[0].getCons.car.getNum == 1) + assert(alist[0].getCons.cdr.getNum == 2) + assert(alist[1].getCons.cdr.getStr == "foo") + + # Generator: + var j = convertSexp([true, false, "foobar", [1, 2, "baz"]]) + assert($j == """(t nil "foobar" (1 2 "baz"))""") diff --git a/tools/nimsuggest/tester.nim b/tools/nimsuggest/tester.nim new file mode 100644 index 000000000..c90afe3db --- /dev/null +++ b/tools/nimsuggest/tester.nim @@ -0,0 +1,182 @@ +# Tester for nimsuggest. +# Every test file can have a #[!]# comment that is deleted from the input +# before 'nimsuggest' is invoked to ensure this token doesn't make a +# crucial difference for Nim's parser. + +import os, osproc, strutils, streams, re + +type + Test = object + cmd, dest: string + startup: seq[string] + script: seq[(string, string)] + +const + curDir = when defined(windows): "" else: "" + DummyEof = "!EOF!" + +template tpath(): untyped = getAppDir() / "tests" + +proc parseTest(filename: string): Test = + const cursorMarker = "#[!]#" + let nimsug = curDir & addFileExt("nimsuggest", ExeExt) + result.dest = getTempDir() / extractFilename(filename) + result.cmd = nimsug & " --tester " & result.dest + result.script = @[] + result.startup = @[] + var tmp = open(result.dest, fmWrite) + var specSection = 0 + var markers = newSeq[string]() + var i = 1 + for x in lines(filename): + let marker = x.find(cursorMarker)+1 + if marker > 0: + markers.add "\"" & filename & "\";\"" & result.dest & "\":" & $i & ":" & $marker + tmp.writeLine x.replace(cursorMarker, "") + else: + tmp.writeLine x + if x.contains("""""""""): + inc specSection + elif specSection == 1: + if x.startsWith("$nimsuggest"): + result.cmd = x % ["nimsuggest", nimsug, "file", filename] + elif x.startsWith("!"): + if result.cmd.len == 0: + result.startup.add x + else: + result.script.add((x, "")) + elif x.startsWith(">"): + # since 'markers' here are not complete yet, we do the $substitutions + # afterwards + result.script.add((x.substr(1).replaceWord("$path", tpath()), "")) + elif x.len > 0: + # expected output line: + let x = x % ["file", filename] + result.script[^1][1].add x.replace(";;", "\t") & '\L' + # else: ignore empty lines for better readability of the specs + inc i + tmp.close() + # now that we know the markers, substitute them: + for a in mitems(result.script): + a[0] = a[0] % markers + +proc parseCmd(c: string): seq[string] = + # we don't support double quotes for now so that + # we can later support them properly with escapes and stuff. + result = @[] + var i = 0 + var a = "" + while true: + setLen(a, 0) + # eat all delimiting whitespace + while c[i] in {' ', '\t', '\l', '\r'}: inc(i) + case c[i] + of '"': raise newException(ValueError, "double quotes not yet supported: " & c) + of '\'': + var delim = c[i] + inc(i) # skip ' or " + while c[i] != '\0' and c[i] != delim: + add a, c[i] + inc(i) + if c[i] != '\0': inc(i) + of '\0': break + else: + while c[i] > ' ': + add(a, c[i]) + inc(i) + add(result, a) + +proc edit(tmpfile: string; x: seq[string]) = + if x.len != 3 and x.len != 4: + quit "!edit takes two or three arguments" + let f = if x.len >= 4: tpath() / x[3] else: tmpfile + try: + let content = readFile(f) + let newcontent = content.replace(x[1], x[2]) + if content == newcontent: + quit "wrong test case: edit had no effect" + writeFile(f, newcontent) + except IOError: + quit "cannot edit file " & tmpfile + +proc exec(x: seq[string]) = + if x.len != 2: quit "!exec takes one argument" + if execShellCmd(x[1]) != 0: + quit "External program failed " & x[1] + +proc copy(x: seq[string]) = + if x.len != 3: quit "!copy takes two arguments" + let rel = tpath() + copyFile(rel / x[1], rel / x[2]) + +proc del(x: seq[string]) = + if x.len != 2: quit "!del takes one argument" + removeFile(tpath() / x[1]) + +proc runCmd(cmd, dest: string): bool = + result = cmd[0] == '!' + if not result: return + let x = cmd.parseCmd() + case x[0] + of "!edit": + edit(dest, x) + of "!exec": + exec(x) + of "!copy": + copy(x) + of "!del": + del(x) + else: + quit "unkown command: " & cmd + +proc smartCompare(pattern, x: string): bool = + if pattern.contains('*'): + result = match(x, re(escapeRe(pattern).replace("\\x2A","(.*)"), {})) + +proc runTest(filename: string): int = + let s = parseTest filename + for cmd in s.startup: + if not runCmd(cmd, s.dest): + quit "invalid command: " & cmd + let cl = parseCmdLine(s.cmd) + var p = startProcess(command=cl[0], args=cl[1 .. ^1], + options={poStdErrToStdOut, poUsePath, + poInteractive, poDemon}) + let outp = p.outputStream + let inp = p.inputStream + var report = "" + var a = newStringOfCap(120) + try: + # read and ignore anything nimsuggest says at startup: + while outp.readLine(a): + if a == DummyEof: break + for req, resp in items(s.script): + if not runCmd(req, s.dest): + inp.writeLine(req) + inp.flush() + var answer = "" + while outp.readLine(a): + if a == DummyEof: break + answer.add a + answer.add '\L' + if resp != answer and not smartCompare(resp, answer): + report.add "\nTest failed: " & filename + report.add "\n Expected: " & resp + report.add "\n But got: " & answer + finally: + inp.writeLine("quit") + inp.flush() + close(p) + if report.len > 0: + echo report + result = report.len + +proc main() = + var failures = 0 + for x in walkFiles(getAppDir() / "tests/t*.nim"): + echo "Test ", x + failures += runTest(expandFilename(x)) + if failures > 0: + quit 1 + +main() diff --git a/tools/nimsuggest/tests/dep_v1.nim b/tools/nimsuggest/tests/dep_v1.nim new file mode 100644 index 000000000..eae230e85 --- /dev/null +++ b/tools/nimsuggest/tests/dep_v1.nim @@ -0,0 +1,8 @@ + + + + + +type + Foo* = object + x*, y*: int diff --git a/tools/nimsuggest/tests/dep_v2.nim b/tools/nimsuggest/tests/dep_v2.nim new file mode 100644 index 000000000..ab39721c4 --- /dev/null +++ b/tools/nimsuggest/tests/dep_v2.nim @@ -0,0 +1,9 @@ + + + + + +type + Foo* = object + x*, y*: int + z*: string diff --git a/tools/nimsuggest/tests/tdef1.nim b/tools/nimsuggest/tests/tdef1.nim new file mode 100644 index 000000000..960ffad1c --- /dev/null +++ b/tools/nimsuggest/tests/tdef1.nim @@ -0,0 +1,16 @@ +discard """ +$nimsuggest --tester $file +>def $1 +def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +>def $1 +def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100 +""" + +proc hello() string = + ## Return hello + "Hello" + +hel#[!]#lo() + +# v uncompleted id for sug (13,2) +he diff --git a/tools/nimsuggest/tests/tdot1.nim b/tools/nimsuggest/tests/tdot1.nim new file mode 100644 index 000000000..bcd44cd84 --- /dev/null +++ b/tools/nimsuggest/tests/tdot1.nim @@ -0,0 +1,14 @@ +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;$file;;11;;4;;"";;100 +sug;;skField;;y;;int;;$file;;11;;7;;"";;100 +sug;;skProc;;tdot1.main;;proc (f: Foo);;$file;;13;;5;;"";;100 +""" + +type + Foo = object + x, y: int + +proc main(f: Foo) = + f.#[!]# diff --git a/tools/nimsuggest/tests/tdot2.nim b/tools/nimsuggest/tests/tdot2.nim new file mode 100644 index 000000000..a58ac818b --- /dev/null +++ b/tools/nimsuggest/tests/tdot2.nim @@ -0,0 +1,29 @@ +# Test basic editing. We replace the 'false' by 'true' to +# see whether then the z field is suggested. + +const zField = 0i32 + +type + Foo = object + x, y: int + when zField == 1i32: + z: string + +proc main(f: Foo) = + f.#[!]# + +# the tester supports the spec section at the bottom of the file and +# this way, the line numbers more often stay the same +discard """ +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;$file;;8;;4;;"";;100 +sug;;skField;;y;;int;;$file;;8;;7;;"";;100 +sug;;skProc;;tdot2.main;;proc (f: Foo);;$file;;12;;5;;"";;100 +!edit 0i32 1i32 +>sug $1 +sug;;skField;;x;;int;;$file;;8;;4;;"";;100 +sug;;skField;;y;;int;;$file;;8;;7;;"";;100 +sug;;skField;;z;;string;;$file;;10;;6;;"";;100 +sug;;skProc;;tdot2.main;;proc (f: Foo);;$file;;12;;5;;"";;100 +""" diff --git a/tools/nimsuggest/tests/tdot3.nim b/tools/nimsuggest/tests/tdot3.nim new file mode 100644 index 000000000..5badde867 --- /dev/null +++ b/tools/nimsuggest/tests/tdot3.nim @@ -0,0 +1,27 @@ +# Test basic module dependency recompilations. + +import dep + +proc main(f: Foo) = + f.#[!]# + +# the tester supports the spec section at the bottom of the file and +# this way, the line numbers more often stay the same + +discard """ +!copy dep_v1.nim dep.nim +$nimsuggest --tester $file +>sug $1 +sug;;skField;;x;;int;;*dep.nim;;8;;4;;"";;100 +sug;;skField;;y;;int;;*dep.nim;;8;;8;;"";;100 +sug;;skProc;;tdot3.main;;proc (f: Foo);;$file;;5;;5;;"";;100 + +!copy dep_v2.nim dep.nim +>mod $path/dep.nim +>sug $1 +sug;;skField;;x;;int;;*dep.nim;;8;;4;;"";;100 +sug;;skField;;y;;int;;*dep.nim;;8;;8;;"";;100 +sug;;skField;;z;;string;;*dep.nim;;9;;4;;"";;100 +sug;;skProc;;tdot3.main;;proc (f: Foo);;$file;;5;;5;;"";;100 +!del dep.nim +""" diff --git a/tools/nimsuggest/tests/tinclude.nim b/tools/nimsuggest/tests/tinclude.nim new file mode 100644 index 000000000..77492d745 --- /dev/null +++ b/tools/nimsuggest/tests/tinclude.nim @@ -0,0 +1,7 @@ +discard """ +$nimsuggest --tester compiler/nim.nim +>def compiler/semexprs.nim:13:50 +def;;skType;;ast.PSym;;PSym;;*ast.nim;;668;;2;;"";;100 +>def compiler/semexprs.nim:13:50 +def;;skType;;ast.PSym;;PSym;;*ast.nim;;668;;2;;"";;100 +""" diff --git a/tools/nimsuggest/tests/tstrutils.nim b/tools/nimsuggest/tests/tstrutils.nim new file mode 100644 index 000000000..f5cda9505 --- /dev/null +++ b/tools/nimsuggest/tests/tstrutils.nim @@ -0,0 +1,9 @@ +discard """ +$nimsuggest --tester lib/pure/strutils.nim +>def lib/pure/strutils.nim:2300:6 +def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"";;100 +""" + +# Line 2300 in strutils.nim is doAssert and this is unlikely to change +# soon since there are a whole lot of doAsserts there. + |