Merge pull request #2680 from reactormonk/epc

Implements EPC for nim-mode in nimsuggest

4 files changed, 955 insertions, 83 deletions

diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim
index 8285d81d9..3b4aa658d 100644
--- a/compiler/nimsuggest/nimsuggest.nim
+++ b/compiler/nimsuggest/nimsuggest.nim
@@ -9,9 +9,17 @@
 
 ## Nimsuggest is a tool that helps to give editors IDE like capabilities.
 
-import strutils, os, parseopt, parseUtils
+import strutils, os, parseopt, parseutils, sequtils, net
+# Do NOT import suggest. It will lead to wierd bugs with
+# suggestionResultHook, because suggest.nim is included by sigmatch.
+# So we import that one instead.
 import options, commands, modules, sem, passes, passaux, msgs, nimconf,
-  extccomp, condsyms, lists, net, rdstdin
+  extccomp, condsyms, lists, net, rdstdin, sexp, sigmatch, ast
+
+when defined(windows):
+  import winlean
+else:
+  import posix
 
 const Usage = """
 Nimsuggest - Tool to give every editor IDE like capabilities for Nim
@@ -23,17 +31,20 @@ Options:
   --address:HOST          binds to that address, by default ""
   --stdin                 read commands from stdin and write results to
                           stdout instead of using sockets
+  --epc                   use emacs epc mode
 
 The server then listens to the connection and takes line-based commands.
 
 In addition, all command line options of Nim that do not affect code generation
 are supported.
 """
+type
+  Mode = enum mstdin, mtcp, mepc
 
 var
   gPort = 6000.Port
   gAddress = ""
-  gUseStdin: bool
+  gMode: Mode
 
 const
   seps = {':', ';', ' ', '\t'}
@@ -42,6 +53,9 @@ const
          "type 'debug' to toggle debug mode on/off\n" &
          "type 'terse' to toggle terse mode on/off"
 
+type
+  EUnexpectedCommand = object of Exception
+
 proc parseQuoted(cmd: string; outp: var string; start: int): int =
   var i = start
   i += skipWhitespace(cmd, i)
@@ -51,7 +65,96 @@ proc parseQuoted(cmd: string; outp: var string; start: int): int =
     i += parseUntil(cmd, outp, seps, i)
   result = i
 
-proc action(cmd: string) =
+proc sexp(s: IdeCmd): SexpNode = sexp($s)
+
+proc sexp(s: TSymKind): SexpNode = sexp($s)
+
+proc sexp(s: Suggest): SexpNode =
+  # If you change the oder here, make sure to change it over in
+  # nim-mode.el too.
+  result = convertSexp([
+    s.section,
+    s.symkind,
+    s.qualifiedPath.map(newSString),
+    s.filePath,
+    s.forth,
+    s.line,
+    s.column,
+    s.doc
+  ])
+
+proc sexp(s: seq[Suggest]): SexpNode =
+  result = newSList()
+  for sug in s:
+    result.add(sexp(sug))
+
+proc listEPC(): SexpNode =
+  let
+    argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol))
+    docstring = sexp("line starts at 1, column at 0, dirtyfile is optional")
+  result = newSList()
+  for command in ["sug", "con", "def", "use"]:
+    let
+      cmd = sexp(command)
+      methodDesc = newSList()
+    methodDesc.add(cmd)
+    methodDesc.add(argspecs)
+    methodDesc.add(docstring)
+    result.add(methodDesc)
+
+proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int) =
+  gIdeCmd = cmd
+  if cmd == ideUse:
+    modules.resetAllModules()
+  var isKnownFile = true
+  let dirtyIdx = file.fileInfoIdx(isKnownFile)
+
+  if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile)
+  else: msgs.setDirtyFile(dirtyIdx, nil)
+
+  resetModule dirtyIdx
+  if dirtyIdx != gProjectMainIdx:
+    resetModule gProjectMainIdx
+
+  gTrackPos = newLineInfo(dirtyIdx, line, col)
+  gErrorCounter = 0
+  if not isKnownFile:
+    compileProject(dirtyIdx)
+  else:
+    compileProject()
+
+proc executeEPC(cmd: IdeCmd, args: SexpNode) =
+  let
+    file = args[0].getStr
+    line = args[1].getNum
+    column = args[2].getNum
+  var dirtyfile = ""
+  if len(args) > 3:
+    dirtyfile = args[3].getStr(nil)
+  execute(cmd, file, dirtyfile, int(line), int(column))
+
+proc returnEPC(socket: var Socket, uid: BiggestInt, s: SexpNode, return_symbol = "return") =
+  let response = $convertSexp([newSSymbol(return_symbol), uid, s])
+  socket.send(toHex(len(response), 6))
+  socket.send(response)
+
+proc connectToNextFreePort(server: Socket, host: string, start = 30000): int =
+  result = start
+  while true:
+    try:
+      server.bindaddr(Port(result), host)
+      return
+    except OsError:
+      when defined(windows):
+        let checkFor = WSAEADDRINUSE.OSErrorCode
+      else:
+        let checkFor = EADDRINUSE.OSErrorCode
+      if osLastError() != checkFor:
+        raise getCurrentException()
+      else:
+        result += 1
+
+proc parseCmdLine(cmd: string) =
   template toggle(sw) =
     if sw in gGlobalOptions:
       excl(gGlobalOptions, sw)
@@ -69,9 +172,7 @@ proc action(cmd: string) =
   of "sug": gIdeCmd = ideSug
   of "con": gIdeCmd = ideCon
   of "def": gIdeCmd = ideDef
-  of "use":
-    modules.resetAllModules()
-    gIdeCmd = ideUse
+  of "use": gIdeCmd = ideUse
   of "quit": quit()
   of "debug": toggle optIdeDebug
   of "terse": toggle optIdeTerse
@@ -88,35 +189,20 @@ proc action(cmd: string) =
   i += skipWhile(cmd, seps, i)
   i += parseInt(cmd, col, i)
 
-  var isKnownFile = true
-  if orig.len == 0: err()
-  let dirtyIdx = orig.fileInfoIdx(isKnownFile)
-
-  if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile)
-  else: msgs.setDirtyFile(dirtyIdx, nil)
-
-  resetModule dirtyIdx
-  if dirtyIdx != gProjectMainIdx:
-    resetModule gProjectMainIdx
-  gTrackPos = newLineInfo(dirtyIdx, line, col-1)
-  #echo dirtyfile, gDirtyBufferIdx, " project ", gProjectMainIdx
-  gErrorCounter = 0
-  if not isKnownFile:
-    compileProject(dirtyIdx)
-  else:
-    compileProject()
+  execute(gIdeCmd, orig, dirtyfile, line, col-1)
 
 proc serve() =
   # do not stop after the first error:
   msgs.gErrorMax = high(int)
-  if gUseStdin:
+  case gMode:
+  of mstdin:
     echo Help
     var line = ""
     while readLineFromStdin("> ", line):
-      action line
+      parseCmdLine line
       echo ""
       flushFile(stdout)
-  else:
+  of mtcp:
     var server = newSocket()
     server.bindAddr(gPort, gAddress)
     var inp = "".TaintedString
@@ -130,10 +216,55 @@ proc serve() =
       accept(server, stdoutSocket)
 
       stdoutSocket.readLine(inp)
-      action inp.string
+      parseCmdLine inp.string
 
       stdoutSocket.send("\c\L")
       stdoutSocket.close()
+  of mepc:
+    var server = newSocket()
+    let port = connectToNextFreePort(server, "localhost")
+    var inp = "".TaintedString
+    server.listen()
+    echo(port)
+    var client = newSocket()
+    # Wait for connection
+    accept(server, client)
+    while true:
+      var sizeHex = ""
+      if client.recv(sizeHex, 6) != 6:
+        raise newException(ValueError, "didn't get all the hexbytes")
+      var size = 0
+      if parseHex(sizeHex, size) == 0:
+        raise newException(ValueError, "invalid size hex: " & $sizeHex)
+      var messageBuffer = ""
+      if client.recv(messageBuffer, size) != size:
+        raise newException(ValueError, "didn't get all the bytes")
+      let
+        message = parseSexp($messageBuffer)
+        messageType = message[0].getSymbol
+      case messageType:
+      of "call":
+        var results: seq[Suggest] = @[]
+        suggestionResultHook = proc (s: Suggest) =
+          results.add(s)
+
+        let
+          uid = message[1].getNum
+          cmd = parseIdeCmd(message[2].getSymbol)
+          args = message[3]
+        executeEPC(cmd, args)
+        returnEPC(client, uid, sexp(results))
+      of "return":
+        raise newException(EUnexpectedCommand, "no return expected")
+      of "return-error":
+        raise newException(EUnexpectedCommand, "no return expected")
+      of "epc-error":
+        stderr.writeln("recieved epc error: " & $messageBuffer)
+        raise newException(IOError, "epc error")
+      of "methods":
+        returnEPC(client, message[1].getNum, listEPC())
+      else:
+        raise newException(EUnexpectedCommand, "unexpected call: " & messageType)
 
 proc mainCommand =
   registerPass verbosePass
@@ -151,15 +282,22 @@ proc mainCommand =
 
 proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
   var p = parseopt.initOptParser(cmd)
-  while true:
+  while true: 
     parseopt.next(p)
     case p.kind
-    of cmdEnd: break
-    of cmdLongoption, cmdShortOption:
+    of cmdEnd: break 
+    of cmdLongoption, cmdShortOption: 
       case p.key.normalize
-      of "port": gPort = parseInt(p.val).Port
-      of "address": gAddress = p.val
-      of "stdin": gUseStdin = true
+      of "port":
+        gPort = parseInt(p.val).Port
+        gMode = mtcp
+      of "address":
+        gAddress = p.val
+        gMode = mtcp
+      of "stdin": gMode = mstdin
+      of "epc":
+        gMode = mepc
+        gVerbosity = 0          # Port number gotta be first.
       else: processSwitch(pass, p)
     of cmdArgument:
       options.gProjectName = unixToNativePath(p.key)
diff --git a/compiler/options.nim b/compiler/options.nim
index 998ab7781..d07342fce 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -83,11 +83,11 @@ type                          # please make sure we have under 32 options
   TGCMode* = enum             # the selected GC
     gcNone, gcBoehm, gcMarkAndSweep, gcRefc, gcV2, gcGenerational
 
-  TIdeCmd* = enum
+  IdeCmd* = enum
     ideNone, ideSug, ideCon, ideDef, ideUse
 
 var
-  gIdeCmd*: TIdeCmd
+  gIdeCmd*: IdeCmd
 
 const
   ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck,
@@ -395,3 +395,18 @@ template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx
 template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx
 template lnimdbg*: expr = L.fileIdx == gProjectMainIdx
 
+proc parseIdeCmd*(s: string): IdeCmd =
+  case s:
+  of "sug": ideSug
+  of "con": ideCon
+  of "def": ideDef
+  of "use": ideUse
+  else: ideNone
+
+proc `$`*(c: IdeCmd): string =
+  case c:
+  of ideSug: "sug"
+  of ideCon: "con"
+  of ideDef: "def"
+  of ideUse: "use"
+  of ideNone: "none"
diff --git a/compiler/suggest.nim b/compiler/suggest.nim
index 6b168670c..659f1fa16 100644
--- a/compiler/suggest.nim
+++ b/compiler/suggest.nim
@@ -15,57 +15,79 @@ import algorithm, sequtils
 
 const
   sep = '\t'
-  sectionSuggest = "sug"
-  sectionDef = "def"
-  sectionContext = "con"
-  sectionUsage = "use"
+
+type
+  Suggest* = object
+    section*: IdeCmd
+    qualifiedPath*: seq[string]
+    filePath*: string
+    line*: int                   # Starts at 1
+    column*: int                 # Starts at 0
+    doc*: string           # Not escaped (yet)
+    symkind*: TSymKind
+    forth*: string               # XXX TODO object on symkind
+
+var
+  suggestionResultHook*: proc (result: Suggest) {.closure.}
 
 #template sectionSuggest(): expr = "##begin\n" & getStackTrace() & "##end\n"
 
 template origModuleName(m: PSym): string = m.name.s
 
-proc symToStr(s: PSym, isLocal: bool, section: string, li: TLineInfo): string = 
-  result = section
-  result.add(sep)
+proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest = 
+  result.section = parseIdeCmd(section)
   if optIdeTerse in gGlobalOptions:
-    if s.kind in routineKinds:
-      result.add renderTree(s.ast, {renderNoBody, renderNoComments,
-                                    renderDocComments, renderNoPragmas})
-    else:
-      result.add s.name.s
-    result.add(sep)
-    result.add(toFullPath(li))
-    result.add(sep)
-    result.add($toLinenumber(li))
-    result.add(sep)
-    result.add($toColumn(li))
+    result.symkind = s.kind
+    result.filePath = toFullPath(li)
+    result.line = toLinenumber(li)
+    result.column = toColumn(li)
   else:
-    result.add($s.kind)
-    result.add(sep)
+    result.symkind = s.kind
+    result.qualifiedPath = @[]
     if not isLocal and s.kind != skModule:
       let ow = s.owner
       if ow.kind != skModule and ow.owner != nil:
         let ow2 = ow.owner
-        result.add(ow2.origModuleName)
-        result.add('.')
-      result.add(ow.origModuleName)
-      result.add('.')
-    result.add(s.name.s)
-    result.add(sep)
+        result.qualifiedPath.add(ow2.origModuleName)
+      result.qualifiedPath.add(ow.origModuleName)
+    result.qualifiedPath.add(s.name.s)
+
     if s.typ != nil: 
-      result.add(typeToString(s.typ))
-    result.add(sep)
-    result.add(toFullPath(li))
-    result.add(sep)
-    result.add($toLinenumber(li))
-    result.add(sep)
-    result.add($toColumn(li))
-    result.add(sep)
+      result.forth = typeToString(s.typ)
+    else:
+      result.forth = ""
+    result.filePath = toFullPath(li)
+    result.line = toLinenumber(li)
+    result.column = toColumn(li)
     when not defined(noDocgen):
-      result.add(s.extractDocComment.escape)
+      result.doc = s.extractDocComment
+
+proc `$`(suggest: Suggest): string = 
+  result = $suggest.section
+  result.add(sep)
+  result.add($suggest.symkind)
+  result.add(sep)
+  result.add(suggest.qualifiedPath.join("."))
+  result.add(sep)
+  result.add(suggest.forth)
+  result.add(sep)
+  result.add(suggest.filePath)
+  result.add(sep)
+  result.add($suggest.line)
+  result.add(sep)
+  result.add($suggest.column)
+  result.add(sep)
+  when not defined(noDocgen):
+    result.add(suggest.doc.escape)
 
-proc symToStr(s: PSym, isLocal: bool, section: string): string = 
-  result = symToStr(s, isLocal, section, s.info)
+proc symToSuggest(s: PSym, isLocal: bool, section: string): Suggest = 
+  result = symToSuggest(s, isLocal, section, s.info)
+
+proc suggestResult(s: Suggest) =
+  if not isNil(suggestionResultHook):
+    suggestionResultHook(s)
+  else:
+    suggestWriteln($(s))
 
 proc filterSym(s: PSym): bool {.inline.} =
   result = s.kind != skModule
@@ -84,7 +106,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
 
 proc suggestField(c: PContext, s: PSym, outputs: var int) = 
   if filterSym(s) and fieldVisible(c, s):
-    suggestWriteln(symToStr(s, isLocal=true, sectionSuggest))
+    suggestResult(symToSuggest(s, isLocal=true, $ideSug))
     inc outputs
 
 template wholeSymTab(cond, section: expr) {.immediate.} =
@@ -97,7 +119,7 @@ template wholeSymTab(cond, section: expr) {.immediate.} =
     for item in entries:
       let it {.inject.} = item
       if cond:
-        suggestWriteln(symToStr(it, isLocal = isLocal, section))
+        suggestResult(symToSuggest(it, isLocal = isLocal, section))
         inc outputs
 
 proc suggestSymList(c: PContext, list: PNode, outputs: var int) = 
@@ -140,7 +162,7 @@ proc argsFit(c: PContext, candidate: PSym, n, nOrig: PNode): bool =
 
 proc suggestCall(c: PContext, n, nOrig: PNode, outputs: var int) = 
   wholeSymTab(filterSym(it) and nameFits(c, it, n) and argsFit(c, it, n, nOrig),
-              sectionContext)
+              $ideCon)
 
 proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} = 
   if s.typ != nil and sonsLen(s.typ) > 1 and s.typ.sons[1] != nil:
@@ -157,7 +179,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} =
 
 proc suggestOperations(c: PContext, n: PNode, typ: PType, outputs: var int) =
   assert typ != nil
-  wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), sectionSuggest)
+  wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), $ideSug)
 
 proc suggestEverything(c: PContext, n: PNode, outputs: var int) =
   # do not produce too many symbols:
@@ -166,7 +188,7 @@ proc suggestEverything(c: PContext, n: PNode, outputs: var int) =
     if scope == c.topLevelScope: isLocal = false
     for it in items(scope.symbols):
       if filterSym(it):
-        suggestWriteln(symToStr(it, isLocal = isLocal, sectionSuggest))
+        suggestResult(symToSuggest(it, isLocal = isLocal, $ideSug))
         inc outputs
     if scope == c.topLevelScope: break
 
@@ -181,12 +203,12 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
         # all symbols accessible, because we are in the current module:
         for it in items(c.topLevelScope.symbols):
           if filterSym(it): 
-            suggestWriteln(symToStr(it, isLocal=false, sectionSuggest))
+            suggestResult(symToSuggest(it, isLocal=false, $ideSug))
             inc outputs
       else: 
         for it in items(n.sym.tab): 
           if filterSym(it): 
-            suggestWriteln(symToStr(it, isLocal=false, sectionSuggest))
+            suggestResult(symToSuggest(it, isLocal=false, $ideSug))
             inc outputs
     else:
       # fallback:
@@ -263,16 +285,16 @@ var
 proc findUsages(info: TLineInfo; s: PSym) =
   if usageSym == nil and isTracked(info, s.name.s.len):
     usageSym = s
-    suggestWriteln(symToStr(s, isLocal=false, sectionUsage))
+    suggestResult(symToSuggest(s, isLocal=false, $ideUse))
   elif s == usageSym:
     if lastLineInfo != info:
-      suggestWriteln(symToStr(s, isLocal=false, sectionUsage, info))
+      suggestResult(symToSuggest(s, isLocal=false, $ideUse, info))
     lastLineInfo = info
 
 proc findDefinition(info: TLineInfo; s: PSym) =
   if s.isNil: return
   if isTracked(info, s.name.s.len):
-    suggestWriteln(symToStr(s, isLocal=false, sectionDef))
+    suggestResult(symToSuggest(s, isLocal=false, $ideDef))
     suggestQuit()
 
 proc ensureIdx[T](x: var T, y: int) =
diff --git a/lib/pure/sexp.nim b/lib/pure/sexp.nim
new file mode 100644
index 000000000..3c9fbc150
--- /dev/null
+++ b/lib/pure/sexp.nim
@@ -0,0 +1,697 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf, Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import
+  hashes, strutils, lexbase, streams, unicode, macros
+
+type
+  SexpEventKind* = enum  ## enumeration of all events that may occur when parsing
+    sexpError,           ## an error occurred during parsing
+    sexpEof,             ## end of file reached
+    sexpString,          ## a string literal
+    sexpSymbol,          ## a symbol
+    sexpInt,             ## an integer literal
+    sexpFloat,           ## a float literal
+    sexpNil,             ## the value ``nil``
+    sexpDot,             ## the dot to separate car/cdr
+    sexpListStart,       ## start of a list: the ``(`` token
+    sexpListEnd,         ## end of a list: the ``)`` token
+
+  TTokKind = enum        # must be synchronized with SexpEventKind!
+    tkError,
+    tkEof,
+    tkString,
+    tkSymbol,
+    tkInt,
+    tkFloat,
+    tkNil,
+    tkDot,
+    tkParensLe,
+    tkParensRi
+    tkSpace
+
+  SexpError* = enum        ## enumeration that lists all errors that can occur
+    errNone,               ## no error
+    errInvalidToken,       ## invalid token
+    errParensRiExpected,    ## ``)`` expected
+    errQuoteExpected,      ## ``"`` expected
+    errEofExpected,        ## EOF expected
+
+  SexpParser* = object of BaseLexer ## the parser object.
+    a: string
+    tok: TTokKind
+    kind: SexpEventKind
+    err: SexpError
+
+const
+  errorMessages: array [SexpError, string] = [
+    "no error",
+    "invalid token",
+    "')' expected",
+    "'\"' or \"'\" expected",
+    "EOF expected",
+  ]
+  tokToStr: array [TTokKind, string] = [
+    "invalid token",
+    "EOF",
+    "string literal",
+    "symbol",
+    "int literal",
+    "float literal",
+    "nil",
+    ".",
+    "(", ")", "space"
+  ]
+
+proc close*(my: var SexpParser) {.inline.} =
+  ## closes the parser `my` and its associated input stream.
+  lexbase.close(my)
+
+proc str*(my: SexpParser): string {.inline.} =
+  ## returns the character data for the events: ``sexpInt``, ``sexpFloat``,
+  ## ``sexpString``
+  assert(my.kind in {sexpInt, sexpFloat, sexpString})
+  result = my.a
+
+proc getInt*(my: SexpParser): BiggestInt {.inline.} =
+  ## returns the number for the event: ``sexpInt``
+  assert(my.kind == sexpInt)
+  result = parseBiggestInt(my.a)
+
+proc getFloat*(my: SexpParser): float {.inline.} =
+  ## returns the number for the event: ``sexpFloat``
+  assert(my.kind == sexpFloat)
+  result = parseFloat(my.a)
+
+proc kind*(my: SexpParser): SexpEventKind {.inline.} =
+  ## returns the current event type for the SEXP parser
+  result = my.kind
+
+proc getColumn*(my: SexpParser): int {.inline.} =
+  ## get the current column the parser has arrived at.
+  result = getColNumber(my, my.bufpos)
+
+proc getLine*(my: SexpParser): int {.inline.} =
+  ## get the current line the parser has arrived at.
+  result = my.lineNumber
+
+proc errorMsg*(my: SexpParser): string =
+  ## returns a helpful error message for the event ``sexpError``
+  assert(my.kind == sexpError)
+  result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), errorMessages[my.err]]
+
+proc errorMsgExpected*(my: SexpParser, e: string): string =
+  ## returns an error message "`e` expected" in the same format as the
+  ## other error messages
+  result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), e & " expected"]
+
+proc handleHexChar(c: char, x: var int): bool =
+  result = true # Success
+  case c
+  of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
+  of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
+  of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
+  else: result = false # error
+
+proc parseString(my: var SexpParser): TTokKind =
+  result = tkString
+  var pos = my.bufpos + 1
+  var buf = my.buf
+  while true:
+    case buf[pos]
+    of '\0':
+      my.err = errQuoteExpected
+      result = tkError
+      break
+    of '"':
+      inc(pos)
+      break
+    of '\\':
+      case buf[pos+1]
+      of '\\', '"', '\'', '/':
+        add(my.a, buf[pos+1])
+        inc(pos, 2)
+      of 'b':
+        add(my.a, '\b')
+        inc(pos, 2)
+      of 'f':
+        add(my.a, '\f')
+        inc(pos, 2)
+      of 'n':
+        add(my.a, '\L')
+        inc(pos, 2)
+      of 'r':
+        add(my.a, '\C')
+        inc(pos, 2)
+      of 't':
+        add(my.a, '\t')
+        inc(pos, 2)
+      of 'u':
+        inc(pos, 2)
+        var r: int
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        add(my.a, toUTF8(Rune(r)))
+      else:
+        # don't bother with the error
+        add(my.a, buf[pos])
+        inc(pos)
+    of '\c':
+      pos = lexbase.handleCR(my, pos)
+      buf = my.buf
+      add(my.a, '\c')
+    of '\L':
+      pos = lexbase.handleLF(my, pos)
+      buf = my.buf
+      add(my.a, '\L')
+    else:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos # store back
+
+proc parseNumber(my: var SexpParser) =
+  var pos = my.bufpos
+  var buf = my.buf
+  if buf[pos] == '-':
+    add(my.a, '-')
+    inc(pos)
+  if buf[pos] == '.':
+    add(my.a, "0.")
+    inc(pos)
+  else:
+    while buf[pos] in Digits:
+      add(my.a, buf[pos])
+      inc(pos)
+    if buf[pos] == '.':
+      add(my.a, '.')
+      inc(pos)
+  # digits after the dot:
+  while buf[pos] in Digits:
+    add(my.a, buf[pos])
+    inc(pos)
+  if buf[pos] in {'E', 'e'}:
+    add(my.a, buf[pos])
+    inc(pos)
+    if buf[pos] in {'+', '-'}:
+      add(my.a, buf[pos])
+      inc(pos)
+    while buf[pos] in Digits:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos
+
+proc parseSymbol(my: var SexpParser) =
+  var pos = my.bufpos
+  var buf = my.buf
+  if buf[pos] in IdentStartChars:
+    while buf[pos] in IdentChars:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos
+
+proc getTok(my: var SexpParser): TTokKind =
+  setLen(my.a, 0)
+  case my.buf[my.bufpos]
+  of '-', '0'..'9': # numbers that start with a . are not parsed
+                    # correctly.
+    parseNumber(my)
+    if {'.', 'e', 'E'} in my.a:
+      result = tkFloat
+    else:
+      result = tkInt
+  of '"': #" # gotta fix nim-mode
+    result = parseString(my)
+  of '(':
+    inc(my.bufpos)
+    result = tkParensLe
+  of ')':
+    inc(my.bufpos)
+    result = tkParensRi
+  of '\0':
+    result = tkEof
+  of 'a'..'z', 'A'..'Z', '_':
+    parseSymbol(my)
+    if my.a == "nil":
+      result = tkNil
+    else:
+      result = tkSymbol
+  of ' ':
+    result = tkSpace
+    inc(my.bufpos)
+  of '.':
+    result = tkDot
+    inc(my.bufpos)
+  else:
+    inc(my.bufpos)
+    result = tkError
+  my.tok = result
+
+# ------------- higher level interface ---------------------------------------
+
+type
+  SexpNodeKind* = enum ## possible SEXP node types
+    SNil,
+    SInt,
+    SFloat,
+    SString,
+    SSymbol,
+    SList,
+    SCons
+
+  SexpNode* = ref SexpNodeObj ## SEXP node
+  SexpNodeObj* {.acyclic.} = object
+    case kind*: SexpNodeKind
+    of SString:
+      str*: string
+    of SSymbol:
+      symbol*: string
+    of SInt:
+      num*: BiggestInt
+    of SFloat:
+      fnum*: float
+    of SList:
+      elems*: seq[SexpNode]
+    of SCons:
+      car: SexpNode
+      cdr: SexpNode
+    of SNil:
+      discard
+
+  Cons = tuple[car: SexpNode, cdr: SexpNode]
+
+  SexpParsingError* = object of ValueError ## is raised for a SEXP error
+
+proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} =
+  ## raises an `ESexpParsingError` exception.
+  raise newException(SexpParsingError, errorMsgExpected(p, msg))
+
+proc newSString*(s: string): SexpNode {.procvar.}=
+  ## Creates a new `SString SexpNode`.
+  new(result)
+  result.kind = SString
+  result.str = s
+
+proc newSStringMove(s: string): SexpNode =
+  new(result)
+  result.kind = SString
+  shallowCopy(result.str, s)
+
+proc newSInt*(n: BiggestInt): SexpNode {.procvar.} =
+  ## Creates a new `SInt SexpNode`.
+  new(result)
+  result.kind = SInt
+  result.num  = n
+
+proc newSFloat*(n: float): SexpNode {.procvar.} =
+  ## Creates a new `SFloat SexpNode`.
+  new(result)
+  result.kind = SFloat
+  result.fnum  = n
+
+proc newSNil*(): SexpNode {.procvar.} =
+  ## Creates a new `SNil SexpNode`.
+  new(result)
+
+proc newSCons*(car, cdr: SexpNode): SexpNode {.procvar.} =
+  ## Creates a new `SCons SexpNode`
+  new(result)
+  result.kind = SCons
+  result.car = car
+  result.cdr = cdr
+
+proc newSList*(): SexpNode {.procvar.} =
+  ## Creates a new `SList SexpNode`
+  new(result)
+  result.kind = SList
+  result.elems = @[]
+
+proc newSSymbol*(s: string): SexpNode {.procvar.} =
+  new(result)
+  result.kind = SSymbol
+  result.symbol = s
+
+proc newSSymbolMove(s: string): SexpNode =
+  new(result)
+  result.kind = SSymbol
+  shallowCopy(result.symbol, s)
+
+proc getStr*(n: SexpNode, default: string = ""): string =
+  ## Retrieves the string value of a `SString SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SString``.
+  if n.kind != SString: return default
+  else: return n.str
+
+proc getNum*(n: SexpNode, default: BiggestInt = 0): BiggestInt =
+  ## Retrieves the int value of a `SInt SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SInt``.
+  if n.kind != SInt: return default
+  else: return n.num
+
+proc getFNum*(n: SexpNode, default: float = 0.0): float =
+  ## Retrieves the float value of a `SFloat SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SFloat``.
+  if n.kind != SFloat: return default
+  else: return n.fnum
+
+proc getSymbol*(n: SexpNode, default: string = ""): string =
+  ## Retrieves the int value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind != SSymbol: return default
+  else: return n.symbol
+
+proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] =
+  ## Retrieves the int value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind == SNil: return @[]
+  elif n.kind != SList: return default
+  else: return n.elems
+
+proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons =
+  ## Retrieves the cons value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind == SCons: return (n.car, n.cdr)
+  elif n.kind == SList: return (n.elems[0], n.elems[1])
+  else: return defaults
+
+proc sexp*(s: string): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SString SexpNode`.
+  new(result)
+  result.kind = SString
+  result.str = s
+
+proc sexp*(n: BiggestInt): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`.
+  new(result)
+  result.kind = SInt
+  result.num  = n
+
+proc sexp*(n: float): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`.
+  new(result)
+  result.kind = SFloat
+  result.fnum  = n
+
+proc sexp*(b: bool): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SSymbol
+  ## SexpNode` with value t or `SNil SexpNode`.
+  new(result)
+  if b:
+    result.kind = SSymbol
+    result.symbol = "t"
+  else:
+    result.kind = SNil
+
+proc sexp*(elements: openArray[SexpNode]): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SList SexpNode`
+  new(result)
+  result.kind = SList
+  newSeq(result.elems, elements.len)
+  for i, p in pairs(elements): result.elems[i] = p
+
+proc sexp*(s: SexpNode): SexpNode =
+  result = s
+
+proc toSexp(x: NimNode): NimNode {.compiletime.} =
+  case x.kind
+  of nnkBracket:
+    result = newNimNode(nnkBracket)
+    for i in 0 .. <x.len:
+      result.add(toSexp(x[i]))
+
+  else:
+    result = x
+
+  result = prefix(result, "sexp")
+
+macro convertSexp*(x: expr): expr =
+  ## Convert an expression to a SexpNode directly, without having to specify
+  ## `%` for every element.
+  result = toSexp(x)
+
+proc `==`* (a,b: SexpNode): bool =
+  ## Check two nodes for equality
+  if a.isNil:
+    if b.isNil: return true
+    return false
+  elif b.isNil or a.kind != b.kind:
+    return false
+  else:
+    return case a.kind
+    of SString:
+      a.str == b.str
+    of SInt:
+      a.num == b.num
+    of SFloat:
+      a.fnum == b.fnum
+    of SNil:
+      true
+    of SList:
+      a.elems == b.elems
+    of SSymbol:
+      a.symbol == b.symbol
+    of SCons:
+      a.car == b.car and a.cdr == b.cdr
+
+proc hash* (n:SexpNode): THash =
+  ## Compute the hash for a SEXP node
+  case n.kind
+  of SList:
+    result = hash(n.elems)
+  of SInt:
+    result = hash(n.num)
+  of SFloat:
+    result = hash(n.fnum)
+  of SString:
+    result = hash(n.str)
+  of SNil:
+    result = hash(0)
+  of SSymbol:
+    result = hash(n.symbol)
+  of SCons:
+    result = hash(n.car) !& hash(n.cdr)
+
+proc len*(n: SexpNode): int =
+  ## If `n` is a `SList`, it returns the number of elements.
+  ## If `n` is a `JObject`, it returns the number of pairs.
+  ## Else it returns 0.
+  case n.kind
+  of SList: result = n.elems.len
+  else: discard
+
+proc `[]`*(node: SexpNode, index: int): SexpNode =
+  ## Gets the node at `index` in a List. Result is undefined if `index`
+  ## is out of bounds
+  assert(not isNil(node))
+  assert(node.kind == SList)
+  return node.elems[index]
+
+proc add*(father, child: SexpNode) =
+  ## Adds `child` to a SList node `father`.
+  assert father.kind == SList
+  father.elems.add(child)
+
+# ------------- pretty printing ----------------------------------------------
+
+proc indent(s: var string, i: int) =
+  s.add(spaces(i))
+
+proc newIndent(curr, indent: int, ml: bool): int =
+  if ml: return curr + indent
+  else: return indent
+
+proc nl(s: var string, ml: bool) =
+  if ml: s.add("\n")
+
+proc escapeJson*(s: string): string =
+  ## Converts a string `s` to its JSON representation.
+  result = newStringOfCap(s.len + s.len shr 3)
+  result.add("\"")
+  for x in runes(s):
+    var r = int(x)
+    if r >= 32 and r <= 127:
+      var c = chr(r)
+      case c
+      of '"': result.add("\\\"") #" # gotta fix nim-mode
+      of '\\': result.add("\\\\")
+      else: result.add(c)
+    else:
+      result.add("\\u")
+      result.add(toHex(r, 4))
+  result.add("\"")
+
+proc copy*(p: SexpNode): SexpNode =
+  ## Performs a deep copy of `a`.
+  case p.kind
+  of SString:
+    result = newSString(p.str)
+  of SInt:
+    result = newSInt(p.num)
+  of SFloat:
+    result = newSFloat(p.fnum)
+  of SNil:
+    result = newSNil()
+  of SSymbol:
+    result = newSSymbol(p.symbol)
+  of SList:
+    result = newSList()
+    for i in items(p.elems):
+      result.elems.add(copy(i))
+  of SCons:
+    result = newSCons(copy(p.car), copy(p.cdr))
+
+proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true,
+              lstArr = false, currIndent = 0) =
+  case node.kind
+  of SString:
+    if lstArr: result.indent(currIndent)
+    result.add(escapeJson(node.str))
+  of SInt:
+    if lstArr: result.indent(currIndent)
+    result.add($node.num)
+  of SFloat:
+    if lstArr: result.indent(currIndent)
+    result.add($node.fnum)
+  of SNil:
+    if lstArr: result.indent(currIndent)
+    result.add("nil")
+  of SSymbol:
+    if lstArr: result.indent(currIndent)
+    result.add($node.symbol)
+  of SList:
+    if lstArr: result.indent(currIndent)
+    if len(node.elems) != 0:
+      result.add("(")
+      result.nl(ml)
+      for i in 0..len(node.elems)-1:
+        if i > 0:
+          result.add(" ")
+          result.nl(ml) # New Line
+        toPretty(result, node.elems[i], indent, ml,
+            true, newIndent(currIndent, indent, ml))
+      result.nl(ml)
+      result.indent(currIndent)
+      result.add(")")
+    else: result.add("nil")
+  of SCons:
+    if lstArr: result.indent(currIndent)
+    result.add("(")
+    toPretty(result, node.car, indent, ml,
+        true, newIndent(currIndent, indent, ml))
+    result.add(" . ")
+    toPretty(result, node.cdr, indent, ml,
+        true, newIndent(currIndent, indent, ml))
+    result.add(")")
+
+proc pretty*(node: SexpNode, indent = 2): string =
+  ## Converts `node` to its Sexp Representation, with indentation and
+  ## on multiple lines.
+  result = ""
+  toPretty(result, node, indent)
+
+proc `$`*(node: SexpNode): string =
+  ## Converts `node` to its SEXP Representation on one line.
+  result = ""
+  toPretty(result, node, 0, false)
+
+iterator items*(node: SexpNode): SexpNode =
+  ## Iterator for the items of `node`. `node` has to be a SList.
+  assert node.kind == SList
+  for i in items(node.elems):
+    yield i
+
+iterator mitems*(node: var SexpNode): var SexpNode =
+  ## Iterator for the items of `node`. `node` has to be a SList. Items can be
+  ## modified.
+  assert node.kind == SList
+  for i in mitems(node.elems):
+    yield i
+
+proc eat(p: var SexpParser, tok: TTokKind) =
+  if p.tok == tok: discard getTok(p)
+  else: raiseParseErr(p, tokToStr[tok])
+
+proc parseSexp(p: var SexpParser): SexpNode =
+  ## Parses SEXP from a SEXP Parser `p`.
+  case p.tok
+  of tkString:
+    # we capture 'p.a' here, so we need to give it a fresh buffer afterwards:
+    result = newSStringMove(p.a)
+    p.a = ""
+    discard getTok(p)
+  of tkInt:
+    result = newSInt(parseBiggestInt(p.a))
+    discard getTok(p)
+  of tkFloat:
+    result = newSFloat(parseFloat(p.a))
+    discard getTok(p)
+  of tkNil:
+    result = newSNil()
+    discard getTok(p)
+  of tkSymbol:
+    result = newSSymbolMove(p.a)
+    p.a = ""
+    discard getTok(p)
+  of tkParensLe:
+    result = newSList()
+    discard getTok(p)
+    while p.tok notin {tkParensRi, tkDot}:
+      result.add(parseSexp(p))
+      if p.tok != tkSpace: break
+      discard getTok(p)
+    if p.tok == tkDot:
+      eat(p, tkDot)
+      eat(p, tkSpace)
+      result.add(parseSexp(p))
+      result = newSCons(result[0], result[1])
+    eat(p, tkParensRi)
+  of tkSpace, tkDot, tkError, tkParensRi, tkEof:
+    raiseParseErr(p, "(")
+
+proc open*(my: var SexpParser, input: Stream) =
+  ## initializes the parser with an input stream.
+  lexbase.open(my, input)
+  my.kind = sexpError
+  my.a = ""
+
+proc parseSexp*(s: Stream): SexpNode =
+  ## Parses from a buffer `s` into a `SexpNode`.
+  var p: SexpParser
+  p.open(s)
+  discard getTok(p) # read first token
+  result = p.parseSexp()
+  p.close()
+
+proc parseSexp*(buffer: string): SexpNode =
+  ## Parses Sexp from `buffer`.
+  result = parseSexp(newStringStream(buffer))
+
+when isMainModule:
+  let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""")
+  assert(testSexp[0].getNum == 1)
+  assert(testSexp[1][0].getNum == 98)
+  assert(testSexp[2].getElems == @[])
+  assert(testSexp[4].getSymbol == "foobar")
+  assert(testSexp[5].getStr == "foo")
+
+  let alist = parseSexp("""((1 . 2) (2 . "foo"))""")
+  assert(alist[0].getCons.car.getNum == 1)
+  assert(alist[0].getCons.cdr.getNum == 2)
+  assert(alist[1].getCons.cdr.getStr == "foo")
+
+  # Generator:
+  var j = convertSexp([true, false, "foobar", [1, 2, "baz"]])
+  assert($j == """(t nil "foobar" (1 2 "baz"))""")