bugfix: internal error in evalFieldAccess; parseutils.interpolatedFragments optimized; tstringinterp.nim now works

4 files changed, 80 insertions, 100 deletions

diff --git a/lib/core/macros.nim b/lib/core/macros.nim
index e362922c8..c5afcdf17 100755
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -220,13 +220,13 @@ proc toYaml*(n: PNimrodNode): string {.magic: "AstToYaml".}
   ## Provides more detailed, potentially harder to digest information

   ## than `toLisp`

 

-proc parseExpr*(s: string) : expr {.magic: "ParseExprToAst".}

-  ## Compiles the passed string to its AST representation

-  ## Expects a single expression

+proc parseExpr*(s: string): expr {.magic: "ParseExprToAst".}

+  ## Compiles the passed string to its AST representation.

+  ## Expects a single expression.

 

-proc parseStmt*(s: string) : stmt {.magic: "ParseStmtToAst".}

-  ## Compiles the passed string to its AST representation

-  ## Expects one or more statements

+proc parseStmt*(s: string): stmt {.magic: "ParseStmtToAst".}

+  ## Compiles the passed string to its AST representation.

+  ## Expects one or more statements.

 

 proc getAst*(macroOrTemplate: expr): expr {.magic: "ExpandMacroToAst".}

   ## Obtains the AST nodes returned from a macro or template invocation.

diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim
index 3e85a9ee6..df56fdb84 100755
--- a/lib/pure/parsecfg.nim
+++ b/lib/pure/parsecfg.nim
@@ -70,7 +70,7 @@ type
 # implementation
 
 const 
-  SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF'} 
+  SymChars: TCharSet = {'a'..'z', 'A'..'Z', '0'..'9', '_', '\x80'..'\xFF', '.'} 
   
 proc rawGetTok(c: var TCfgParser, tok: var TToken)
 
diff --git a/lib/pure/parseutils.nim b/lib/pure/parseutils.nim
index 1f4e54b96..001872db2 100755
--- a/lib/pure/parseutils.nim
+++ b/lib/pure/parseutils.nim
@@ -266,94 +266,76 @@ proc parseFloat*(s: string, number: var float, start = 0): int {.
   result = parseBiggestFloat(s, bf, start)
   number = bf
   
-proc isEscaped*(s: string, pos: int) : bool =
-  assert pos >= 0 and pos < s.len
-
-  var
-    backslashes = 0
-    j = pos - 1
-
-  while j >= 0:
-    if s[j] == '\\':
-      inc backslashes
-      dec j
-    else:
-      break
-
-  return backslashes mod 2 != 0
-
 type
-  TInterpolatedKind* = enum
-    ikString, ikExpr
-
-  TInterpStrFragment* = tuple[kind: TInterpolatedKind, value: string]
-
-iterator interpolatedFragments*(s: string): TInterpStrFragment =
-  var 
-    i = 0
-    tokenStart = 0
-
-  proc token(kind: TInterpolatedKind, value: string): TInterpStrFragment =
-    result.kind = kind
-    result.value = value
-
-  while i < s.len:
-    # The $ sign marks the start of an interpolation.
-    #
-    # It's followed either by a varialbe name or an opening bracket 
-    # (so it should be before the end of the string)
-    # if the dollar sign is escaped, don't trigger interpolation
-    if s[i] == '$' and i < (s.len - 1) and not isEscaped(s, i):
-      # Interpolation starts here.
-      # Return any string that we've ran over so far.
-      if i != tokenStart:
-        yield token(ikString, s[tokenStart..i-1])
-
-      var next = s[i+1]
-      if next == '{':
-        # Complex expression: ${foo(bar) in {1..100}}
-        # Find closing braket, while respecting any nested brackets
-        inc i
-        tokenStart = i + 1
-
-        var
-          brackets = {'{', '}'}
-          nestingCount = 1
-        
-        while i < s.len:
-          inc i, skipUntil(s, brackets, i+1) + 1
-          
-          if not isEscaped(s, i):
-            if s[i] == '}':
-              dec nestingCount
-              if nestingCount == 0: break
-            else:
-              inc nestingCount
-
-        yield token(ikExpr, s[tokenStart..(i-1)])
-
-        tokenStart = i + 1
-        
+  TInterpolatedKind* = enum  ## describes for `interpolatedFragments`
+                             ## which part of the interpolated string is
+                             ## yielded; for example in "str$var${expr}"
+    ikStr,                   ## ``str`` part of the interpolated string 
+    ikVar,                   ## ``var`` part of the interpolated string
+    ikExpr                   ## ``expr`` part of the interpolated string
+

+iterator interpolatedFragments*(s: string): tuple[kind: TInterpolatedKind,
+  value: string] =

+  ## Tokenizes the string `s` into substrings for interpolation purposes.

+  ##

+  ## Example:

+  ##

+  ## .. code-block:: nimrod

+  ##   for k, v in interpolatedFragments("  $this is ${an  example}  "):

+  ##     echo "(", k, ", \"", v, "\")"

+  ##

+  ## Results in:

+  ##

+  ## .. code-block:: nimrod

+  ##   (ikString, "  ")

+  ##   (ikExpr, "this")

+  ##   (ikString, " is ")

+  ##   (ikExpr, "an  example")

+  ##   (ikString, "  ")

+  var i = 0
+  var kind: TInterpolatedKind

+  while true:

+    var j = i

+    if s[j] == '$' and s[j+1] != '$':
+      if s[j+1] == '{':
+        inc j, 2
+        var nesting = 0
+        while true:
+          case s[j]
+          of '{': inc nesting
+          of '}':
+            if nesting == 0: 
+              inc j
+              break
+            dec nesting
+          of '\0':
+            raise newException(EInvalidValue, 
+              "Expected closing '}': " & s[i..s.len])
+          else: nil
+          inc j
+        inc i, 2 # skip ${
+        kind = ikExpr
+      elif s[j+1] in IdentStartChars:
+        inc j, 2
+        while s[j] in IdentChars: inc(j)
+        inc i # skip $
+        kind = ikVar
       else:
-        tokenStart = i + 1
-        var identifier = parseIdent(s, i+1)
-        
-        if identifier.len >  0:
-          inc i, identifier.len
-
-          yield token(ikExpr, s[tokenStart..i])
-
-          tokenStart = i + 1
-
-        else:
-          raise newException(EInvalidValue, "Unable to parse a varible name at " & s[i..s.len])
-       
-    inc i
-  #end while
+        raise newException(EInvalidValue, 
+          "Unable to parse a varible name at " & s[i..s.len])
+    else:
+      while j < s.len and (s[j] != '$' or s[j+1] == '$'): inc j
+      kind = ikStr
+    if j > i:
+      # do not copy the trailing } for ikExpr:

+      yield (kind, substr(s, i, j-1-ord(kind == ikExpr)))

+    else:

+      break

+    i = j

+
+when isMainModule:
+  for k, v in interpolatedFragments("$test{}  $this is ${an{  example}}  "):

+    echo "(", k, ", \"", v, "\")"

 
-  # We've reached the end of the string without finding a new interpolation.
-  # Return the last fragment at string.
-  if i != tokenStart:
-    yield token(ikString, s[tokenStart..i])
 
 {.pop.}
diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim
index 9f7e01693..53aa34c83 100755
--- a/lib/pure/strutils.nim
+++ b/lib/pure/strutils.nim
@@ -640,8 +640,7 @@ proc join*(a: openArray[string], sep: string): string {.
   if len(a) > 0:

     var L = sep.len * (a.len-1)

     for i in 0..high(a): inc(L, a[i].len)

-    result = newString(L)

-    setLen(result, 0)

+    result = newStringOfCap(L)

     add(result, a[0])

     for i in 1..high(a):

       add(result, sep)

@@ -655,8 +654,7 @@ proc join*(a: openArray[string]): string {.
   if len(a) > 0:

     var L = 0

     for i in 0..high(a): inc(L, a[i].len)

-    result = newString(L)

-    setLen(result, 0)

+    result = newStringOfCap(L)

     for i in 0..high(a): add(result, a[i])

   else:

     result = ""

@@ -867,9 +865,9 @@ proc validIdentifier*(s: string): bool {.noSideEffect,
 

 proc editDistance*(a, b: string): int {.noSideEffect,

   rtl, extern: "nsuEditDistance".} =

-  ## returns the edit distance between `a` and `b`. This uses the Levenshtein

-  ## distance algorithm with only a linear memory overhead. This implementation

-  ## is highly optimized!

+  ## returns the edit distance between `a` and `b`. This uses the 
+  ## `Levenshtein`:idx: distance algorithm with only a linear memory overhead.
+  ## This implementation is highly optimized!

   var len1 = a.len

   var len2 = b.len

   if len1 > len2: