diff options
Diffstat (limited to 'lib/impure/re.nim')
| -rw-r--r-- | lib/impure/re.nim | 276 |
1 files changed, 173 insertions, 103 deletions
diff --git a/lib/impure/re.nim b/lib/impure/re.nim index fe3ea96ff..053c6ab55 100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -17,6 +17,10 @@ when defined(js): ## C library. This means that your application will depend on the PCRE ## library's licence when using this module, which should not be a problem ## though. +## +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). +## ## PCRE's licence follows: ## ## .. include:: ../../doc/regexprs.txt @@ -32,21 +36,24 @@ runnableExamples: # can't match start of string since we're starting at 1 import - pcre, strutils, rtarrays + std/[pcre, strutils, rtarrays] + +when defined(nimPreviewSlimSystem): + import std/syncio const MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. - ## This limit still exists for ``replacef`` and ``parallelReplace``. + ## This limit still exists for `replacef` and `parallelReplace`. type RegexFlag* = enum ## options for regular expressions - reIgnoreCase = 0, ## do caseless matching - reMultiLine = 1, ## ``^`` and ``$`` match newlines within data - reDotAll = 2, ## ``.`` matches anything including NL - reExtended = 3, ## ignore whitespace and ``#`` comments - reStudy = 4 ## study the expression (may be omitted if the - ## expression will be used only once) + reIgnoreCase = 0, ## do caseless matching + reMultiLine = 1, ## `^` and `$` match newlines within data + reDotAll = 2, ## `.` matches anything including NL + reExtended = 3, ## ignore whitespace and `#` comments + reStudy = 4 ## study the expression (may be omitted if the + ## expression will be used only once) RegexDesc = object h: ptr Pcre @@ -58,10 +65,16 @@ type ## is raised if the pattern is no valid regular expression. when defined(gcDestructors): - proc `=destroy`(x: var RegexDesc) = - pcre.free_substring(cast[cstring](x.h)) - if not isNil(x.e): - pcre.free_study(x.e) + when defined(nimAllowNonVarDestructor): + proc `=destroy`(x: RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) + else: + proc `=destroy`(x: var RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = var e: ref RegexError @@ -79,7 +92,7 @@ proc rawCompile(pattern: string, flags: cint): ptr Pcre = proc finalizeRegEx(x: Regex) = # XXX This is a hack, but PCRE does not export its "free" function properly. - # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``). + # Sigh. The hack relies on PCRE's implementation (see `pcre_get.c`). # Fortunately the implementation is unlikely to change. pcre.free_substring(cast[cstring](x.h)) if not isNil(x.e): @@ -89,8 +102,8 @@ proc re*(s: string, flags = {reStudy}): Regex = ## Constructor of regular expressions. ## ## Note that Nim's - ## extended raw string literals support the syntax ``re"[abc]"`` as - ## a short form for ``re(r"[abc]")``. Also note that since this + ## extended raw string literals support the syntax `re"[abc]"` as + ## a short form for `re(r"[abc]")`. Also note that since this ## compiles the regular expression, which is expensive, you should ## avoid putting it directly in the arguments of the functions like ## the examples show below if you plan to use it a lot of times, as @@ -141,13 +154,20 @@ proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``buf`` - ## (where ``buf`` has length ``bufSize`` and is not necessarily ``'\0'`` terminated), + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), ## and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``(-1,0)`` is returned. + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -163,20 +183,31 @@ proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], start = 0): tuple[first, last: int] {.inline.} = - ## returns the starting position and end position of ``pattern`` in ``s`` - ## and the captured substrings in the array ``matches``. + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. ## If it does not match, nothing - ## is written into ``matches`` and ``(-1,0)`` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[tuple[first, last: int]], - start = 0, bufSize = 0): tuple[first, last: int] = - ## returns the starting position and end position of ``pattern`` in ``buf`` - ## (where ``buf`` has length ``bufSize`` and is not necessarily ``'\0'`` terminated), - ## and the captured substrings in the array ``matches``. - ## If it does not match, nothing is written into ``matches`` and - ## ``(-1,0)`` is returned. + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -193,17 +224,38 @@ proc findBounds*(buf: cstring, pattern: Regex, proc findBounds*(s: string, pattern: Regex, matches: var openArray[tuple[first, last: int]], start = 0): tuple[first, last: int] {.inline.} = - ## returns the starting position and end position of ``pattern`` in ``s`` - ## and the captured substrings in the array ``matches``. - ## If it does not match, nothing is written into ``matches`` and - ## ``(-1,0)`` is returned. - result = findBounds(cstring(s), pattern, matches, start, s.len) + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) proc findBounds*(buf: cstring, pattern: Regex, start = 0, bufSize: int): tuple[first, last: int] = - ## returns the ``first`` and ``last`` position of ``pattern`` in ``buf``, - ## where ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). - ## If it does not match, ``(-1,0)`` is returned. + ## returns the `first` and `last` position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `(-1,0)` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData @@ -214,13 +266,14 @@ proc findBounds*(buf: cstring, pattern: Regex, proc findBounds*(s: string, pattern: Regex, start = 0): tuple[first, last: int] {.inline.} = - ## returns the ``first`` and ``last`` position of ``pattern`` in ``s``. - ## If it does not match, ``(-1,0)`` is returned. + ## returns the `first` and `last` position of `pattern` in `s`. + ## If it does not match, `(-1,0)` is returned. ## ## Note: there is a speed improvement if the matches do not need to be captured. runnableExamples: assert findBounds("01234abc89", re"abc") == (5,7) - result = findBounds(cstring(s), pattern, start, s.len) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = var @@ -233,21 +286,25 @@ proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint) proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], start = 0): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. ## runnableExamples: @@ -257,24 +314,25 @@ proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = result = matchOrFind(cstring(s), pattern, start.cint, s.len.cint, pcre.ANCHORED) proc matchLen*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int {.inline.} = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, ``-1`` is returned. Note that a match length + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. result = matchOrFind(buf, pattern, start.cint, bufSize, pcre.ANCHORED) proc match*(s: string, pattern: Regex, start = 0): bool {.inline.} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern``. + ## returns `true` if `s[start..]` matches the `pattern`. result = matchLen(cstring(s), pattern, start, s.len) != -1 proc match*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is + ## returns `true` if `s[start..]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is ## returned. ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. runnableExamples: - import sequtils + import std/sequtils var matches: array[2, string] if match("abcdefg", re"c(d)ef(g)", matches, 2): doAssert toSeq(matches) == @["d", "g"] @@ -282,19 +340,23 @@ proc match*(s: string, pattern: Regex, matches: var openArray[string], proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], start = 0, bufSize: int): bool {.inline.} = - ## returns ``true`` if ``buf[start..<bufSize]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is + ## returns `true` if `buf[start..<bufSize]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is ## returned. - ## ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = matchLen(buf, pattern, matches, start, bufSize) != -1 proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], - start = 0, bufSize = 0): int = - ## returns the starting position of ``pattern`` in ``buf`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``-1`` is returned. - ## ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). + start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var rtarray = initRtArray[cint]((matches.len+1)*3) rawMatches = rtarray.getRawData @@ -310,15 +372,17 @@ proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], proc find*(s: string, pattern: Regex, matches: var openArray[string], start = 0): int {.inline.} = - ## returns the starting position of ``pattern`` in ``s`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and ``-1`` is returned. + ## returns the starting position of `pattern` in `s` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. result = find(cstring(s), pattern, matches, start, s.len) proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = - ## returns the starting position of ``pattern`` in ``buf``, - ## where ``buf`` has length ``bufSize`` (not necessarily ``'\0'`` terminated). - ## If it does not match, ``-1`` is returned. + ## returns the starting position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `-1` is returned. var rtarray = initRtArray[cint](3) rawMatches = rtarray.getRawData @@ -328,8 +392,8 @@ proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = return rawMatches[0] proc find*(s: string, pattern: Regex, start = 0): int {.inline.} = - ## returns the starting position of ``pattern`` in ``s``. If it does not - ## match, ``-1`` is returned. We start the scan at `start`. + ## returns the starting position of `pattern` in `s`. If it does not + ## match, `-1` is returned. We start the scan at `start`. runnableExamples: doAssert find("abcdefg", re"cde") == 2 doAssert find("abcdefg", re"abc") == 0 @@ -360,7 +424,7 @@ iterator findAll*(s: string, pattern: Regex, start = 0): string = i = b iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string = - ## Yields all matching `substrings` of ``s`` that match ``pattern``. + ## Yields all matching `substrings` of `s` that match `pattern`. ## ## Note that since this is an iterator you should not modify the string you ## are iterating over: bad things could happen. @@ -381,25 +445,22 @@ iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string i = b proc findAll*(s: string, pattern: Regex, start = 0): seq[string] {.inline.} = - ## returns all matching `substrings` of ``s`` that match ``pattern``. - ## If it does not match, @[] is returned. + ## returns all matching `substrings` of `s` that match `pattern`. + ## If it does not match, `@[]` is returned. result = @[] for x in findAll(s, pattern, start): result.add x -when not defined(nimhygiene): - {.pragma: inject.} - template `=~` *(s: string, pattern: Regex): untyped = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: + ## This calls `match` with an implicit declared `matches` array that + ## can be used in the scope of the `=~` call: runnableExamples: proc parse(line: string): string = if line =~ re"\s*(\w+)\s*\=\s*(\w+)": # matches a key=value pair: result = $(matches[0], matches[1]) elif line =~ re"\s*(\#.*)": # matches a comment - # note that the implicit ``matches`` array is different from 1st branch + # note that the implicit `matches` array is different from 1st branch result = $(matches[0],) - else: doAssert false + else: raiseAssert "unreachable" doAssert not declared(matches) doAssert parse("NAME = LENA") == """("NAME", "LENA")""" doAssert parse(" # comment ... ") == """("# comment ... ",)""" @@ -411,12 +472,14 @@ template `=~` *(s: string, pattern: Regex): untyped = # ------------------------- more string handling ------------------------------ proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = - ## same as ``find(s, pattern, start) >= 0`` + ## same as `find(s, pattern, start) >= 0` return find(s, pattern, start) >= 0 proc contains*(s: string, pattern: Regex, matches: var openArray[string], start = 0): bool {.inline.} = - ## same as ``find(s, pattern, matches, start) >= 0`` + ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 proc startsWith*(s: string, prefix: Regex): bool {.inline.} = @@ -429,25 +492,29 @@ proc endsWith*(s: string, suffix: Regex): bool {.inline.} = if matchLen(s, suffix, i) == s.len - i: return true proc replace*(s: string, sub: Regex, by = ""): string = - ## Replaces ``sub`` in ``s`` by the string ``by``. Captures cannot be - ## accessed in ``by``. + ## Replaces `sub` in `s` by the string `by`. Captures cannot be + ## accessed in `by`. runnableExamples: doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)") == "; " doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" result = "" var prev = 0 + var flags = int32(0) while prev < s.len: - var match = findBounds(s, sub, prev) + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 if match.first < 0: break add(result, substr(s, prev, match.first-1)) add(result, by) - if match.last + 1 == prev: break + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART prev = match.last + 1 add(result, substr(s, prev)) proc replacef*(s: string, sub: Regex, by: string): string = - ## Replaces ``sub`` in ``s`` by the string ``by``. Captures can be accessed in ``by`` - ## with the notation ``$i`` and ``$#`` (see strutils.\`%\`). + ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` + ## with the notation `$i` and `$#` (see strutils.\`%\`). runnableExamples: doAssert "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == "var1<-keykey; var2<-key2key2" @@ -465,7 +532,7 @@ proc replacef*(s: string, sub: Regex, by: string): string = proc multiReplace*(s: string, subs: openArray[ tuple[pattern: Regex, repl: string]]): string = - ## Returns a modified copy of ``s`` with the substitutions in ``subs`` + ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" var i = 0 @@ -485,36 +552,39 @@ proc multiReplace*(s: string, subs: openArray[ proc transformFile*(infile, outfile: string, subs: openArray[tuple[pattern: Regex, repl: string]]) = - ## reads in the file ``infile``, performs a parallel replacement (calls - ## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an + ## reads in the file `infile`, performs a parallel replacement (calls + ## `parallelReplace`) and writes back to `outfile`. Raises `IOError` if an ## error occurs. This is supposed to be used for quick scripting. - var x = readFile(infile).string + var x = readFile(infile) writeFile(outfile, x.multiReplace(subs)) iterator split*(s: string, sep: Regex; maxsplit = -1): string = - ## Splits the string ``s`` into substrings. + ## Splits the string `s` into substrings. ## - ## Substrings are separated by the regular expression ``sep`` - ## (and the portion matched by ``sep`` is not returned). + ## Substrings are separated by the regular expression `sep` + ## (and the portion matched by `sep` is not returned). runnableExamples: - import sequtils + import std/sequtils doAssert toSeq(split("00232this02939is39an22example111", re"\d+")) == @["", "this", "is", "an", "example", ""] var last = 0 var splits = maxsplit - var x: int + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 while last <= len(s): var first = last var sepLen = 1 + if x == 0: + inc(last) while last < len(s): x = matchLen(s, sep, last) if x >= 0: sepLen = x break inc(last) - if x == 0: - if last >= len(s): break - inc last if splits == 0: last = len(s) yield substr(s, first, last-1) if splits == 0: break @@ -522,14 +592,14 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string = inc(last, sepLen) proc split*(s: string, sep: Regex, maxsplit = -1): seq[string] {.inline.} = - ## Splits the string ``s`` into a seq of substrings. + ## Splits the string `s` into a seq of substrings. ## - ## The portion matched by ``sep`` is not returned. + ## The portion matched by `sep` is not returned. result = @[] for x in split(s, sep, maxsplit): result.add x proc escapeRe*(s: string): string = - ## escapes ``s`` so that it is matched verbatim when used as a regular + ## escapes `s` so that it is matched verbatim when used as a regular ## expression. result = "" for c in items(s): |